for (FlowStep v: flowStepGraph.vertexSet()) { graph.addVertex(v); for (ProcessEdge e: flowStepGraph.edgeSet()) { graph.addEdge(e.getSourceProcessID(), e.getSinkProcessID());
protected FlowNodeGraph createFlowNodeGraph( FlowStepFactory flowStepFactory, FlowElementGraph flowElementGraph, Map<ElementGraph, List<? extends ElementGraph>> pipelineSubGraphsMap, List<? extends ElementGraph> nodeSubGraphs ) { return new FlowNodeGraph( flowStepFactory.getFlowNodeFactory(), flowElementGraph, nodeSubGraphs, pipelineSubGraphsMap ); } }
@Override public Iterator<Process> getTopologicalIterator() { return getOrderedTopologicalIterator( new Comparator<Process>() { @Override public int compare( Process lhs, Process rhs ) { return Integer.valueOf( lhs.getSubmitPriority() ).compareTo( rhs.getSubmitPriority() ); } } ); }
private EdgeValues( TezConfiguration config, ProcessEdge processEdge ) { this.config = config; this.flowElement = processEdge.getFlowElement(); this.ordinals = processEdge.getSourceProvidedOrdinals(); this.resolvedKeyFieldsMap = processEdge.getResolvedKeyFields(); this.resolvedSortFieldsMap = processEdge.getResolvedSortFields(); this.resolvedValueFieldsMap = processEdge.getResolvedValueFields(); }
private void addInputOutputMapping( JobConf conf, FlowNode flowNode ) { FlowNodeGraph flowNodeGraph = getFlowNodeGraph(); Set<ProcessEdge> incomingEdges = flowNodeGraph.incomingEdgesOf( flowNode ); for( ProcessEdge processEdge : incomingEdges ) conf.set( "cascading.node.source." + processEdge.getFlowElementID(), processEdge.getSourceProcessID() ); Set<ProcessEdge> outgoingEdges = flowNodeGraph.outgoingEdgesOf( flowNode ); for( ProcessEdge processEdge : outgoingEdges ) conf.set( "cascading.node.sink." + processEdge.getFlowElementID(), processEdge.getSinkProcessID() ); }
private void applyEdgeAnnotations( ProcessEdge processEdge, EdgeValues edgeValues ) { processEdge.addEdgeAnnotation( edgeValues.movementType ); processEdge.addEdgeAnnotation( edgeValues.sourceType ); processEdge.addEdgeAnnotation( edgeValues.schedulingType ); }
public ProcessEdge( Process sourceProcess, FlowElement flowElement, Process sinkProcess ) { this( sourceProcess.getElementGraph(), flowElement, sinkProcess.getElementGraph() ); this.sourceProcessID = sourceProcess.getID(); this.sinkProcessID = sinkProcess.getID(); }
@Override public List<ElementGraph> getElementGraphs( Scope scope ) { List<Process> elementProcesses = getElementProcesses( scope ); List<ElementGraph> elementGraphs = new ArrayList<>(); for( Process elementProcess : elementProcesses ) elementGraphs.add( elementProcess.getElementGraph() ); return elementGraphs; }
@Override public Set<FlowElement> getAllSourceElements() { Set<FlowElement> results = createIdentitySet(); for( Process process : vertexSet() ) results.addAll( process.getSourceElements() ); return results; }
@Override public Set<ElementGraph> getIdentityElementGraphs() { Set<ElementGraph> results = createIdentitySet(); for( Process process : getIdentityProcesses() ) results.add( process.getElementGraph() ); return results; }
public EnumMultiMap<FlowElement> getAnnotations() { EnumMultiMap<FlowElement> annotations = new EnumMultiMap<>(); for( Process process : vertexSet() ) { ElementGraph elementGraph = process.getElementGraph(); if( elementGraph instanceof AnnotatedGraph ) annotations.addAll( ( (AnnotatedGraph) elementGraph ).getAnnotations() ); } return annotations; }
/** * Method writeDOT writes this element graph to a DOT file for easy visualization and debugging. * * @param filename of type String */ @Override public void writeDOT( String filename ) { printProcessGraph( filename ); }
@Override public String getVertexName( ProcessModel processModel ) { return "" + processModel.getOrdinal(); } }
private EdgeValues( TezConfiguration config, ProcessEdge processEdge ) { this.config = config; this.flowElement = processEdge.getFlowElement(); this.ordinals = processEdge.getSourceProvidedOrdinals(); this.resolvedKeyFieldsMap = processEdge.getResolvedKeyFields(); this.resolvedSortFieldsMap = processEdge.getResolvedSortFields(); this.resolvedValueFieldsMap = processEdge.getResolvedValueFields(); }
private void applyEdgeAnnotations( ProcessEdge processEdge, EdgeValues edgeValues ) { processEdge.addEdgeAnnotation( edgeValues.movementType ); processEdge.addEdgeAnnotation( edgeValues.sourceType ); processEdge.addEdgeAnnotation( edgeValues.schedulingType ); }
@Override public Iterator<Process> getOrdinalTopologicalIterator() { return getOrderedTopologicalIterator( new Comparator<Process>() { @Override public int compare( Process lhs, Process rhs ) { return Integer.valueOf( lhs.getOrdinal() ).compareTo( rhs.getOrdinal() ); } } ); }
public Iterator<FlowNode> getOrderedTopologicalIterator() { return super.getOrderedTopologicalIterator( FLOW_NODE_COMPARATOR ); } }