public static boolean isSinkOf(Tap tap, Flow flow) { for(Object t: flow.getSinksCollection()) { if(t==tap) return true; } return false; } }
/** * Method addFlow adds a new {@link cascading.flow.Flow} instance that is intended to participate in a {@link Cascade}. * * @param flow of Flow * @return CascadeDef */ public CascadeDef addFlow( Flow flow ) { if( flow == null ) return this; if( flows.containsKey( flow.getName() ) ) throw new CascadeException( "all flow names must be unique, found duplicate: " + flow.getName() ); Collection<Tap> sinks = flow.getSinksCollection(); for( Tap sink : sinks ) { String fullIdentifier = sink.getFullIdentifier( flow.getConfig() ); for( Flow existingFlow : flows.values() ) { Collection<Tap> existingSinks = existingFlow.getSinksCollection(); for( Tap existingSink : existingSinks ) { if( fullIdentifier.equals( existingSink.getFullIdentifier( existingFlow.getConfig() ) ) ) throw new CascadeException( "the flow: " + flow.getName() + ", has a sink identifier: " + fullIdentifier + ", in common with the flow: " + existingFlow.getName() ); } } } flows.put( flow.getName(), flow ); return this; }
private void makeGraph( Flow[] flows ) { for( Flow flow : flows ) { LinkedList<Tap> sources = new LinkedList<Tap>( flow.getSourcesCollection() ); LinkedList<Tap> sinks = new LinkedList<Tap>( flow.getSinksCollection() ); sinks.addAll( flow.getCheckpointsCollection() ); unwrapCompositeTaps( sources ); unwrapCompositeTaps( sinks ); for( Tap source : sources ) addVertex( getVertex( flow, source ) ); for( Tap sink : sinks ) addVertex( getVertex( flow, sink ) ); for( Tap source : sources ) { for( Tap sink : sinks ) addEdgeFor( flow, source, sink ); } } }
@Test public void testCheckpointTapCascade() throws IOException { if( !getPlatform().isMapReduce() ) return; getPlatform().copyFromLocal( inputFileIps ); String path = "checkpoint"; Flow first = firstFlow( path + "/first", false ); Flow second = secondFlow( first.getSink(), path + "/second" ); Flow third = thirdCheckpointFlow( second.getSink(), path + "/third" ); Flow fourth = fourthFlow( (Tap) third.getCheckpoints().values().iterator().next(), path + "/fourth" ); Cascade cascade = new CascadeConnector( getProperties() ).connect( fourth, second, third, first ); cascade.start(); cascade.complete(); validateLength( fourth, 20 ); assertTrue( cascade.getHeadFlows().contains( first ) ); assertTrue( cascade.getSourceTaps().containsAll( first.getSourcesCollection() ) ); assertTrue( cascade.getIntermediateTaps().containsAll( third.getCheckpointsCollection() ) ); assertTrue( cascade.getCheckpointsTaps().containsAll( third.getCheckpointsCollection() ) ); assertTrue( cascade.getTailFlows().contains( fourth ) ); assertTrue( cascade.getSinkTaps().containsAll( fourth.getSinksCollection() ) ); }
@Test public void testCheckpointTapCascade() throws IOException { if( !getPlatform().isMapReduce() ) return; getPlatform().copyFromLocal( inputFileIps ); String path = "checkpoint"; Flow first = firstFlow( path + "/first", false ); Flow second = secondFlow( first.getSink(), path + "/second" ); Flow third = thirdCheckpointFlow( second.getSink(), path + "/third" ); Flow fourth = fourthFlow( (Tap) third.getCheckpoints().values().iterator().next(), path + "/fourth" ); Cascade cascade = new CascadeConnector( getProperties() ).connect( fourth, second, third, first ); cascade.start(); cascade.complete(); validateLength( fourth, 20 ); assertTrue( cascade.getHeadFlows().contains( first ) ); assertTrue( cascade.getSourceTaps().containsAll( first.getSourcesCollection() ) ); assertTrue( cascade.getIntermediateTaps().containsAll( third.getCheckpointsCollection() ) ); assertTrue( cascade.getCheckpointsTaps().containsAll( third.getCheckpointsCollection() ) ); assertTrue( cascade.getTailFlows().contains( fourth ) ); assertTrue( cascade.getSinkTaps().containsAll( fourth.getSinksCollection() ) ); }
@Test public void testSimpleCascade() throws IOException { getPlatform().copyFromLocal( inputFileIps ); String path = "simple"; Flow first = firstFlow( path + "/first", false ); Flow second = secondFlow( first.getSink(), path + "/second" ); Flow third = thirdFlow( second.getSink(), path + "/third" ); Flow fourth = fourthFlow( third.getSink(), path + "/fourth" ); Cascade cascade = new CascadeConnector( getProperties() ).connect( fourth, second, third, first ); cascade.start(); cascade.complete(); validateLength( fourth, 20 ); assertTrue( cascade.getHeadFlows().contains( first ) ); assertTrue( cascade.getSourceTaps().containsAll( first.getSourcesCollection() ) ); assertTrue( cascade.getTailFlows().contains( fourth ) ); assertTrue( cascade.getSinkTaps().containsAll( fourth.getSinksCollection() ) ); }
@Test public void testSimpleCascade() throws IOException { getPlatform().copyFromLocal( inputFileIps ); String path = "simple"; Flow first = firstFlow( path + "/first", false ); Flow second = secondFlow( first.getSink(), path + "/second" ); Flow third = thirdFlow( second.getSink(), path + "/third" ); Flow fourth = fourthFlow( third.getSink(), path + "/fourth" ); Cascade cascade = new CascadeConnector( getProperties() ).connect( fourth, second, third, first ); cascade.start(); cascade.complete(); validateLength( fourth, 20 ); assertTrue( cascade.getHeadFlows().contains( first ) ); assertTrue( cascade.getSourceTaps().containsAll( first.getSourcesCollection() ) ); assertTrue( cascade.getTailFlows().contains( fourth ) ); assertTrue( cascade.getSinkTaps().containsAll( fourth.getSinksCollection() ) ); }