public void iterate( Set<Hfs> taps, MultiSourceTap value ) { Iterator<Tap> childTaps = value.getChildTaps(); while( childTaps.hasNext() ) accumulate( taps, childTaps.next() ); }
public void accumulate( Set<Hfs> taps, Tap value ) { if( value == null ) return; if( value instanceof Hfs ) taps.add( (Hfs) value ); else if( value instanceof PartitionTap ) taps.add( (Hfs) ( (PartitionTap) value ).getParent() ); else if( value instanceof MultiSourceTap ) iterate( taps, (MultiSourceTap) value ); else throw new IllegalArgumentException( "unsupprted Tap type: " + value.getClass().getName() ); }
@Override public void onStarting( Flow flow ) { Map<String, Tap> sources = flow.getSources(); for( Map.Entry<String, Tap> entry : sources.entrySet() ) { String key = entry.getKey(); Tap value = entry.getValue(); Set<Hfs> taps = Util.createIdentitySet(); accumulate( taps, value ); for( Hfs tap : taps ) { if( !testExists( flow, tap ) ) throw new FlowException( "cannot start flow: " + flow.getName() + ", _SUCCESS file missing in tap: '" + key + "', at: " + value.getIdentifier() ); } } }
@Test public void testFailOnMissingSuccessFlowListener() throws Exception { getPlatform().copyFromLocal( inputFileLower ); FlowListener listener = new FailOnMissingSuccessFlowListener(); Hfs source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileLower ); Hfs success = new Hfs( new TextLine(), getOutputPath( "withsuccess" ), SinkMode.REPLACE ); Hfs without = new Hfs( new TextLine(), getOutputPath( "withoutsuccess" ), SinkMode.REPLACE ); Hfs sink = new Hfs( new TextLine(), getOutputPath( "final" ), SinkMode.REPLACE ); Flow firstFlow = getPlatform().getFlowConnector( getProperties() ).connect( source, success, new Pipe( "lower" ) ); firstFlow.addListener( listener ); firstFlow.complete(); Flow secondFlow = getPlatform().getFlowConnector( getProperties() ).connect( success, without, new Pipe( "lower" ) ); secondFlow.addListener( listener ); secondFlow.complete(); Hfs successTap = new Hfs( new TextLine(), new Path( without.getPath(), "_SUCCESS" ).toString() ); assertTrue( successTap.deleteResource( getPlatform().getFlowProcess() ) ); Flow thirdFlow = getPlatform().getFlowConnector( getProperties() ).connect( without, sink, new Pipe( "lower" ) ); thirdFlow.addListener( listener ); try { thirdFlow.complete(); fail( "listener did not fail flow" ); } catch( FlowException exception ) { // do nothing } } }
@Override public void onStarting( Flow flow ) { Map<String, Tap> sources = flow.getSources(); for( Map.Entry<String, Tap> entry : sources.entrySet() ) { String key = entry.getKey(); Tap value = entry.getValue(); Set<Hfs> taps = Util.createIdentitySet(); accumulate( taps, value ); for( Hfs tap : taps ) { if( !testExists( flow, tap ) ) throw new FlowException( "cannot start flow: " + flow.getName() + ", _SUCCESS file missing in tap: '" + key + "', at: " + value.getIdentifier() ); } } }
@Test public void testFailOnMissingSuccessFlowListener() throws Exception { getPlatform().copyFromLocal( inputFileLower ); FlowListener listener = new FailOnMissingSuccessFlowListener(); Hfs source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), inputFileLower ); Hfs success = new Hfs( new TextLine(), getOutputPath( "withsuccess" ), SinkMode.REPLACE ); Hfs without = new Hfs( new TextLine(), getOutputPath( "withoutsuccess" ), SinkMode.REPLACE ); Hfs sink = new Hfs( new TextLine(), getOutputPath( "final" ), SinkMode.REPLACE ); Flow firstFlow = getPlatform().getFlowConnector( getProperties() ).connect( source, success, new Pipe( "lower" ) ); firstFlow.addListener( listener ); firstFlow.complete(); Flow secondFlow = getPlatform().getFlowConnector( getProperties() ).connect( success, without, new Pipe( "lower" ) ); secondFlow.addListener( listener ); secondFlow.complete(); Hfs successTap = new Hfs( new TextLine(), new Path( without.getPath(), "_SUCCESS" ).toString() ); assertTrue( successTap.deleteResource( getPlatform().getFlowProcess() ) ); Flow thirdFlow = getPlatform().getFlowConnector( getProperties() ).connect( without, sink, new Pipe( "lower" ) ); thirdFlow.addListener( listener ); try { thirdFlow.complete(); fail( "listener did not fail flow" ); } catch( FlowException exception ) { // do nothing } } }
public void accumulate( Set<Hfs> taps, Tap value ) { if( value == null ) return; if( value instanceof Hfs ) taps.add( (Hfs) value ); else if( value instanceof PartitionTap ) taps.add( (Hfs) ( (PartitionTap) value ).getParent() ); else if( value instanceof MultiSourceTap ) iterate( taps, (MultiSourceTap) value ); else throw new IllegalArgumentException( "unsupprted Tap type: " + value.getClass().getName() ); }
public void iterate( Set<Hfs> taps, MultiSourceTap value ) { Iterator<Tap> childTaps = value.getChildTaps(); while( childTaps.hasNext() ) accumulate( taps, childTaps.next() ); }