public void attachFlowStep( JobConf jobConf ) { if( completeCalled ) throw new IllegalStateException( "cannot attach new FlowStep after complete() has been called" ); addFlowStep( createMapReduceFlowStep( jobConf ) ); }
@Override protected boolean spawnSteps() throws InterruptedException, ExecutionException { // continue to spawn jobs until no longer required while( !stop && throwable == null ) { if( !blockingContinuePollingSteps() ) return true; if( isInfoEnabled() ) { logInfo( "updated" ); for( Tap source : getSourcesCollection() ) logInfo( " source: " + source ); for( Tap sink : getSinksCollection() ) logInfo( " sink: " + sink ); } // will not return until all current steps are complete, or one failed if( !super.spawnSteps() ) return false; } return true; }
protected Collection<MapReduceFlowStep> updateWithFlowSteps( Collection<MapReduceFlowStep> flowSteps ) { if( flowSteps.isEmpty() ) return flowSteps; FlowStepGraph flowStepGraph = getOrCreateFlowStepGraph(); updateFlowStepGraph( flowStepGraph, flowSteps ); setFlowElementGraph( asFlowElementGraph( platformInfo, flowStepGraph ) ); removeListeners( getSourcesCollection() ); removeListeners( getSinksCollection() ); removeListeners( getTrapsCollection() ); // re-adds listeners setSources( flowStepGraph.getSourceTapsMap() ); setSinks( flowStepGraph.getSinkTapsMap() ); setTraps( flowStepGraph.getTrapsMap() ); // this mirrors BaseFlow#initialize() initSteps(); if( flowStats == null ) flowStats = createPrepareFlowStats(); // must be last if( !isJobsMapInitialized() ) initializeNewJobsMap(); else updateJobsMap(); initializeChildStats(); return flowSteps; }
protected void initializeFrom( List<JobConf> jobConfs ) { List<MapReduceFlowStep> steps = new ArrayList<>(); for( JobConf jobConf : jobConfs ) steps.add( createMapReduceFlowStep( jobConf ) ); updateWithFlowSteps( steps ); }
@Test public void testFlowLazy() throws IOException { getPlatform().copyFromLocal( inputFileApache ); String outputPath1 = getOutputPath( "flowTest1" ); String outputPath2 = getOutputPath( "flowTest2" ); String outputPath3 = getOutputPath( "flowTest3" ); remove( outputPath1, true ); remove( outputPath2, true ); remove( outputPath3, true ); JobConf defaultConf = (JobConf) ( (BaseHadoopPlatform) getPlatform() ).getConfiguration(); JobConf conf1 = createJob( defaultConf, "mr1", InputData.inputFileApache, outputPath1 ); JobConf conf2 = createJob( defaultConf, "mr2", outputPath1, outputPath2 ); JobConf conf3 = createJob( defaultConf, "mr3", outputPath2, outputPath3 ); validateLength( new Hfs( new TextLine(), InputData.inputFileApache ).openForRead( new HadoopFlowProcess( defaultConf ) ), 10 ); MultiMapReduceFlow flow = new MultiMapReduceFlow( "mrflow", conf1 ); flow.start(); Util.safeSleep( 3000 ); flow.attachFlowStep( conf2 ); Util.safeSleep( 3000 ); flow.attachFlowStep( conf3 ); flow.complete(); validateLength( new Hfs( new TextLine(), outputPath1 ).openForRead( new HadoopFlowProcess( defaultConf ) ), 10 ); Collection<Tap> sinks = flow.getSinks().values(); assertEquals( 1, sinks.size() ); String identifier = sinks.iterator().next().getIdentifier(); assertEquals( "flowTest3", identifier.substring( identifier.lastIndexOf( '/' ) + 1 ) ); }
@Test(expected = IllegalStateException.class) public void testFlowLazyFail() throws IOException { getPlatform().copyFromLocal( inputFileApache ); String outputPath1 = getOutputPath( "flowTest1" ); String outputPath2 = getOutputPath( "flowTest2" ); remove( outputPath1, true ); remove( outputPath2, true ); JobConf defaultConf = (JobConf) ( (BaseHadoopPlatform) getPlatform() ).getConfiguration(); JobConf conf1 = createJob( defaultConf, "mr1", InputData.inputFileApache, outputPath1 ); JobConf conf2 = createJob( defaultConf, "mr2", outputPath1, outputPath2 ); validateLength( new Hfs( new TextLine(), InputData.inputFileApache ).openForRead( new HadoopFlowProcess( defaultConf ) ), 10 ); MultiMapReduceFlow flow = new MultiMapReduceFlow( "mrflow", conf1 ); flow.complete(); flow.attachFlowStep( conf2 ); }
@Test public void testFlow() throws IOException { getPlatform().copyFromLocal( inputFileApache ); String outputPath1 = getOutputPath( "flowTest1" ); String outputPath2 = getOutputPath( "flowTest2" ); String outputPath3 = getOutputPath( "flowTest3" ); remove( outputPath1, true ); remove( outputPath2, true ); remove( outputPath3, true ); JobConf defaultConf = (JobConf) ( (BaseHadoopPlatform) getPlatform() ).getConfiguration(); JobConf conf1 = createJob( defaultConf, "mr1", InputData.inputFileApache, outputPath1 ); JobConf conf2 = createJob( defaultConf, "mr2", outputPath1, outputPath2 ); JobConf conf3 = createJob( defaultConf, "mr3", outputPath2, outputPath3 ); MultiMapReduceFlow flow = new MultiMapReduceFlow( "mrflow", conf1, conf2, conf3 ); validateLength( new Hfs( new TextLine(), InputData.inputFileApache ).openForRead( new HadoopFlowProcess( defaultConf ) ), 10 ); flow.complete(); validateLength( new Hfs( new TextLine(), outputPath1 ).openForRead( new HadoopFlowProcess( defaultConf ) ), 10 ); Collection<Tap> sinks = flow.getSinks().values(); assertEquals( 1, sinks.size() ); String identifier = sinks.iterator().next().getIdentifier(); assertEquals( "flowTest3", identifier.substring( identifier.lastIndexOf( '/' ) + 1 ) ); }
protected FlowStepGraph getOrCreateFlowStepGraph() { FlowStepGraph flowStepGraph = getFlowStepGraph(); if( flowStepGraph == null ) { flowStepGraph = new FlowStepGraph(); setFlowStepGraph( flowStepGraph ); } return flowStepGraph; }
protected boolean blockingContinuePollingSteps() { synchronized( lock ) { // block until queue has items, or complete is called while( queuedSteps.isEmpty() && !completeCalled ) { try { lock.wait(); } catch( InterruptedException exception ) { // do nothing } } updateWithFlowSteps( queuedSteps ).clear(); } if( getEligibleJobsSize() != 0 ) // new ones were added return true; return !completeCalled; }
@Test public void testFlowLazy() throws IOException { getPlatform().copyFromLocal( inputFileApache ); String outputPath1 = getOutputPath( "flowTest1" ); String outputPath2 = getOutputPath( "flowTest2" ); String outputPath3 = getOutputPath( "flowTest3" ); remove( outputPath1, true ); remove( outputPath2, true ); remove( outputPath3, true ); JobConf defaultConf = (JobConf) ( (BaseHadoopPlatform) getPlatform() ).getConfiguration(); JobConf conf1 = createJob( defaultConf, "mr1", InputData.inputFileApache, outputPath1 ); JobConf conf2 = createJob( defaultConf, "mr2", outputPath1, outputPath2 ); JobConf conf3 = createJob( defaultConf, "mr3", outputPath2, outputPath3 ); validateLength( new Hfs( new TextLine(), InputData.inputFileApache ).openForRead( new HadoopFlowProcess( defaultConf ) ), 10 ); MultiMapReduceFlow flow = new MultiMapReduceFlow( "mrflow", conf1 ); flow.start(); Util.safeSleep( 3000 ); flow.attachFlowStep( conf2 ); Util.safeSleep( 3000 ); flow.attachFlowStep( conf3 ); flow.complete(); validateLength( new Hfs( new TextLine(), outputPath1 ).openForRead( new HadoopFlowProcess( defaultConf ) ), 10 ); Collection<Tap> sinks = flow.getSinks().values(); assertEquals( 1, sinks.size() ); String identifier = sinks.iterator().next().getIdentifier(); assertEquals( "flowTest3", identifier.substring( identifier.lastIndexOf( '/' ) + 1 ) ); }
@Test(expected = IllegalStateException.class) public void testFlowLazyFail() throws IOException { getPlatform().copyFromLocal( inputFileApache ); String outputPath1 = getOutputPath( "flowTest1" ); String outputPath2 = getOutputPath( "flowTest2" ); remove( outputPath1, true ); remove( outputPath2, true ); JobConf defaultConf = (JobConf) ( (BaseHadoopPlatform) getPlatform() ).getConfiguration(); JobConf conf1 = createJob( defaultConf, "mr1", InputData.inputFileApache, outputPath1 ); JobConf conf2 = createJob( defaultConf, "mr2", outputPath1, outputPath2 ); validateLength( new Hfs( new TextLine(), InputData.inputFileApache ).openForRead( new HadoopFlowProcess( defaultConf ) ), 10 ); MultiMapReduceFlow flow = new MultiMapReduceFlow( "mrflow", conf1 ); flow.complete(); flow.attachFlowStep( conf2 ); }
@Test public void testFlow() throws IOException { getPlatform().copyFromLocal( inputFileApache ); String outputPath1 = getOutputPath( "flowTest1" ); String outputPath2 = getOutputPath( "flowTest2" ); String outputPath3 = getOutputPath( "flowTest3" ); remove( outputPath1, true ); remove( outputPath2, true ); remove( outputPath3, true ); JobConf defaultConf = (JobConf) ( (BaseHadoopPlatform) getPlatform() ).getConfiguration(); JobConf conf1 = createJob( defaultConf, "mr1", InputData.inputFileApache, outputPath1 ); JobConf conf2 = createJob( defaultConf, "mr2", outputPath1, outputPath2 ); JobConf conf3 = createJob( defaultConf, "mr3", outputPath2, outputPath3 ); MultiMapReduceFlow flow = new MultiMapReduceFlow( "mrflow", conf1, conf2, conf3 ); validateLength( new Hfs( new TextLine(), InputData.inputFileApache ).openForRead( new HadoopFlowProcess( defaultConf ) ), 10 ); flow.complete(); validateLength( new Hfs( new TextLine(), outputPath1 ).openForRead( new HadoopFlowProcess( defaultConf ) ), 10 ); Collection<Tap> sinks = flow.getSinks().values(); assertEquals( 1, sinks.size() ); String identifier = sinks.iterator().next().getIdentifier(); assertEquals( "flowTest3", identifier.substring( identifier.lastIndexOf( '/' ) + 1 ) ); }
protected void initializeFrom( List<JobConf> jobConfs ) { List<MapReduceFlowStep> steps = new ArrayList<>(); for( JobConf jobConf : jobConfs ) steps.add( createMapReduceFlowStep( jobConf ) ); updateWithFlowSteps( steps ); }
protected FlowStepGraph getOrCreateFlowStepGraph() { FlowStepGraph flowStepGraph = getFlowStepGraph(); if( flowStepGraph == null ) { flowStepGraph = new FlowStepGraph(); setFlowStepGraph( flowStepGraph ); } return flowStepGraph; }
protected boolean blockingContinuePollingSteps() { synchronized( lock ) { // block until queue has items, or complete is called while( queuedSteps.isEmpty() && !completeCalled ) { try { lock.wait(); } catch( InterruptedException exception ) { // do nothing } } updateWithFlowSteps( queuedSteps ).clear(); } if( getEligibleJobsSize() != 0 ) // new ones were added return true; return !completeCalled; }
protected Collection<MapReduceFlowStep> updateWithFlowSteps( Collection<MapReduceFlowStep> flowSteps ) { if( flowSteps.isEmpty() ) return flowSteps; FlowStepGraph flowStepGraph = getOrCreateFlowStepGraph(); updateFlowStepGraph( flowStepGraph, flowSteps ); setFlowElementGraph( asFlowElementGraph( platformInfo, flowStepGraph ) ); removeListeners( getSourcesCollection() ); removeListeners( getSinksCollection() ); removeListeners( getTrapsCollection() ); // re-adds listeners setSources( flowStepGraph.getSourceTapsMap() ); setSinks( flowStepGraph.getSinkTapsMap() ); setTraps( flowStepGraph.getTrapsMap() ); // this mirrors BaseFlow#initialize() initSteps(); if( flowStats == null ) flowStats = createPrepareFlowStats(); // must be last if( !isJobsMapInitialized() ) initializeNewJobsMap(); else updateJobsMap(); initializeChildStats(); return flowSteps; }
@Override protected boolean spawnSteps() throws InterruptedException, ExecutionException { // continue to spawn jobs until no longer required while( !stop && throwable == null ) { if( !blockingContinuePollingSteps() ) return true; if( isInfoEnabled() ) { logInfo( "updated" ); for( Tap source : getSourcesCollection() ) logInfo( " source: " + source ); for( Tap sink : getSinksCollection() ) logInfo( " sink: " + sink ); } // will not return until all current steps are complete, or one failed if( !super.spawnSteps() ) return false; } return true; }
public void attachFlowStep( JobConf jobConf ) { if( completeCalled ) throw new IllegalStateException( "cannot attach new FlowStep after complete() has been called" ); addFlowStep( createMapReduceFlowStep( jobConf ) ); }