public FlowStats( Flow flow, ClientState clientState ) { super( flow.getName(), clientState ); this.flow = flow; }
public String getName() { return flow.getName(); }
public String getEdgeName( BaseFlow.FlowHolder object ) { return object.flow.getName().replaceAll( "\"", "\'" ).replaceAll( "\n", "\\\\n" ); // fix for newlines in graphviz } }
private String makeName( Flow[] flows ) { String[] names = new String[ flows.length ]; for( int i = 0; i < flows.length; i++ ) names[ i ] = flows[ i ].getName(); return Util.join( names, "+" ); } }
@Override public List<Flow> findFlows( String regex ) { List<Flow> flows = new ArrayList<Flow>(); for( Flow flow : getFlows() ) { if( flow.getName().matches( regex ) ) flows.add( flow ); } return flows; }
public void setFlow( Flow<Config> flow ) { this.flow = flow; this.flowID = flow.getID(); this.flowName = flow.getName(); }
public void stop() { if( LOG.isInfoEnabled() ) logInfo( "stopping flow: " + flow.getName() ); stop = true; if( flow != null ) flow.stop(); }
/** * Method addFlow adds a new {@link cascading.flow.Flow} instance that is intended to participate in a {@link Cascade}. * * @param flow of Flow * @return CascadeDef */ public CascadeDef addFlow( Flow flow ) { if( flow == null ) return this; if( flows.containsKey( flow.getName() ) ) throw new CascadeException( "all flow names must be unique, found duplicate: " + flow.getName() ); Collection<Tap> sinks = flow.getSinksCollection(); for( Tap sink : sinks ) { String fullIdentifier = sink.getFullIdentifier( flow.getConfig() ); for( Flow existingFlow : flows.values() ) { Collection<Tap> existingSinks = existingFlow.getSinksCollection(); for( Tap existingSink : existingSinks ) { if( fullIdentifier.equals( existingSink.getFullIdentifier( existingFlow.getConfig() ) ) ) throw new CascadeException( "the flow: " + flow.getName() + ", has a sink identifier: " + fullIdentifier + ", in common with the flow: " + existingFlow.getName() ); } } } flows.put( flow.getName(), flow ); return this; }
formatter.format("\n").format(StringUtils.repeat("=", 90)).format("\n"); formatter.format("Counters for ").format(flow.getName() == null ? "unnamed flow" : "flow " + flow.getName()).format("\n") .format(" with input ").format(prettyTaps(flow.getSources())).format("\n") .format(" and output ").format(prettyTaps(flow.getSinks())).format("\n");
private void initializeNewJobsMap() { synchronized( jobsMap ) { // keep topo order TopologicalOrderIterator<Flow, Integer> topoIterator = flowGraph.getTopologicalIterator(); while( topoIterator.hasNext() ) { Flow flow = topoIterator.next(); cascadeStats.addFlowStats( flow.getFlowStats() ); CascadeJob job = new CascadeJob( flow ); jobsMap.put( flow.getName(), job ); List<CascadeJob> predecessors = new ArrayList<CascadeJob>(); for( Flow predecessor : Graphs.predecessorListOf( flowGraph, flow ) ) predecessors.add( (CascadeJob) jobsMap.get( predecessor.getName() ) ); job.init( predecessors ); } } }
@Override public void onStarting( Flow flow ) { Map<String, Tap> sources = flow.getSources(); for( Map.Entry<String, Tap> entry : sources.entrySet() ) { String key = entry.getKey(); Tap value = entry.getValue(); Set<Hfs> taps = Util.createIdentitySet(); accumulate( taps, value ); for( Hfs tap : taps ) { if( !testExists( flow, tap ) ) throw new FlowException( "cannot start flow: " + flow.getName() + ", _SUCCESS file missing in tap: '" + key + "', at: " + value.getIdentifier() ); } } }
@Override public void onStarting( Flow flow ) { Map<String, Tap> sources = flow.getSources(); for( Map.Entry<String, Tap> entry : sources.entrySet() ) { String key = entry.getKey(); Tap value = entry.getValue(); Set<Hfs> taps = Util.createIdentitySet(); accumulate( taps, value ); for( Hfs tap : taps ) { if( !testExists( flow, tap ) ) throw new FlowException( "cannot start flow: " + flow.getName() + ", _SUCCESS file missing in tap: '" + key + "', at: " + value.getIdentifier() ); } } }
private void makeGraph( IdentifierGraph identifierGraph ) { Set<String> identifiers = identifierGraph.vertexSet(); int count = 0; for( String source : identifiers ) { if( LOG.isDebugEnabled() ) LOG.debug( "handling flow source: {}", source ); List<String> sinks = Graphs.successorListOf( identifierGraph, source ); for( String sink : sinks ) { if( LOG.isDebugEnabled() ) LOG.debug( "handling flow path: {} -> {}", source, sink ); Flow flow = identifierGraph.getEdge( source, sink ).flow; addVertex( flow ); Set<BaseFlow.FlowHolder> previous = identifierGraph.incomingEdgesOf( source ); for( BaseFlow.FlowHolder previousFlow : previous ) { addVertex( previousFlow.flow ); if( getEdge( previousFlow.flow, flow ) != null ) continue; if( !addEdge( previousFlow.flow, flow, count++ ) ) throw new CascadeException( "unable to add path between: " + previousFlow.flow.getName() + " and: " + flow.getName() ); } } } } }
logInfo( "starting flow: " + flow.getName() ); logInfo( "skipping flow: " + flow.getName() ); logInfo( "completed flow: " + flow.getName() ); logWarn( "flow failed: " + flow.getName(), exception ); CascadeException cascadeException = new CascadeException( "flow failed: " + flow.getName(), exception );
private void addEdgeFor( Flow flow, Tap source, Tap sink ) { try { addEdge( getVertex( flow, source ), getVertex( flow, sink ), ( (BaseFlow) flow ).getHolder() ); } catch( IllegalArgumentException exception ) { throw new CascadeException( "no loops allowed in cascade, flow: " + flow.getName() + ", source: " + source + ", sink: " + sink ); } }