protected void buildGraph( FlowStepFactory flowStepFactory, FlowElementGraph flowElementGraph, Map<ElementGraph, List<? extends ElementGraph>> nodeSubGraphsMap, Map<ElementGraph, List<? extends ElementGraph>> pipelineSubGraphsMap ) { for( ElementGraph stepSubGraph : nodeSubGraphsMap.keySet() ) { List<? extends ElementGraph> nodeSubGraphs = nodeSubGraphsMap.get( stepSubGraph ); FlowNodeGraph flowNodeGraph = createFlowNodeGraph( flowStepFactory, flowElementGraph, pipelineSubGraphsMap, nodeSubGraphs ); EnumMultiMap<FlowElement> annotations = flowNodeGraph.getAnnotations(); // pull up annotations if( !annotations.isEmpty() ) stepSubGraph = new AnnotatedDecoratedElementGraph( stepSubGraph, annotations ); FlowStep flowStep = flowStepFactory.createFlowStep( stepSubGraph, flowNodeGraph ); addVertex( flowStep ); } bindEdges(); }
protected FlowStepGraph makeStepGraph( JobConf jobConf ) { FlowStepGraph flowStepGraph = new FlowStepGraph(); Tap sink = getSinksCollection().iterator().next(); FlowStep<JobConf> step = createFlowStep( jobConf, sink ); flowStepGraph.addVertex( step ); return flowStepGraph; }
protected Collection<MapReduceFlowStep> updateWithFlowSteps( Collection<MapReduceFlowStep> flowSteps ) { if( flowSteps.isEmpty() ) return flowSteps; FlowStepGraph flowStepGraph = getOrCreateFlowStepGraph(); updateFlowStepGraph( flowStepGraph, flowSteps ); setFlowElementGraph( asFlowElementGraph( platformInfo, flowStepGraph ) ); removeListeners( getSourcesCollection() ); removeListeners( getSinksCollection() ); removeListeners( getTrapsCollection() ); // re-adds listeners setSources( flowStepGraph.getSourceTapsMap() ); setSinks( flowStepGraph.getSinkTapsMap() ); setTraps( flowStepGraph.getTrapsMap() ); // this mirrors BaseFlow#initialize() initSteps(); if( flowStats == null ) flowStats = createPrepareFlowStats(); // must be last if( !isJobsMapInitialized() ) initializeNewJobsMap(); else updateJobsMap(); initializeChildStats(); return flowSteps; }
public FlowStepGraph( FlowStepFactory flowStepFactory, FlowElementGraph flowElementGraph, Map<ElementGraph, List<? extends ElementGraph>> nodeSubGraphsMap, Map<ElementGraph, List<? extends ElementGraph>> pipelineSubGraphsMap ) { buildGraph( flowStepFactory, flowElementGraph, nodeSubGraphsMap, pipelineSubGraphsMap ); Iterator<FlowStep> iterator = getTopologicalIterator(); int ordinal = 0; int size = vertexSet().size(); while( iterator.hasNext() ) { BaseFlowStep flowStep = (BaseFlowStep) iterator.next(); flowStep.setOrdinal( ordinal++ ); flowStep.setName( flowStepFactory.makeFlowStepName( flowStep, size, flowStep.getOrdinal() ) ); } }
protected FlowStepGraph getOrCreateFlowStepGraph() { FlowStepGraph flowStepGraph = getFlowStepGraph(); if( flowStepGraph == null ) { flowStepGraph = new FlowStepGraph(); setFlowStepGraph( flowStepGraph ); } return flowStepGraph; }
protected void initSteps() { if( flowStepGraph == null ) return; Set<FlowStep> flowSteps = flowStepGraph.vertexSet(); for( FlowStep flowStep : flowSteps ) { ( (BaseFlowStep) flowStep ).setFlow( this ); Set<FlowNode> flowNodes = flowStep.getFlowNodeGraph().vertexSet(); for( FlowNode flowNode : flowNodes ) ( (BaseFlowNode) flowNode ).setFlowStep( flowStep ); } }
protected FlowStepGraph makeStepGraph( JobConf jobConf ) { FlowStepGraph flowStepGraph = new FlowStepGraph(); Tap sink = getSinksCollection().iterator().next(); FlowStep<JobConf> step = createFlowStep( jobConf, sink ); flowStepGraph.addVertex( step ); return flowStepGraph; }
protected Collection<MapReduceFlowStep> updateWithFlowSteps( Collection<MapReduceFlowStep> flowSteps ) { if( flowSteps.isEmpty() ) return flowSteps; FlowStepGraph flowStepGraph = getOrCreateFlowStepGraph(); updateFlowStepGraph( flowStepGraph, flowSteps ); setFlowElementGraph( asFlowElementGraph( platformInfo, flowStepGraph ) ); removeListeners( getSourcesCollection() ); removeListeners( getSinksCollection() ); removeListeners( getTrapsCollection() ); // re-adds listeners setSources( flowStepGraph.getSourceTapsMap() ); setSinks( flowStepGraph.getSinkTapsMap() ); setTraps( flowStepGraph.getTrapsMap() ); // this mirrors BaseFlow#initialize() initSteps(); if( flowStats == null ) flowStats = createPrepareFlowStats(); // must be last if( !isJobsMapInitialized() ) initializeNewJobsMap(); else updateJobsMap(); initializeChildStats(); return flowSteps; }
protected FlowStepGraph getOrCreateFlowStepGraph() { FlowStepGraph flowStepGraph = getFlowStepGraph(); if( flowStepGraph == null ) { flowStepGraph = new FlowStepGraph(); setFlowStepGraph( flowStepGraph ); } return flowStepGraph; }
Map<ElementGraph, List<? extends ElementGraph>> nodeToPipeline = ruleResult.getNodeToPipelineGraphMap(); FlowStepGraph flowStepGraph = new FlowStepGraph( getFlowStepFactory(), finalFlowElementGraph, stepToNodes, nodeToPipeline );