graph.addEdge(e.getSourceProcessID(), e.getSinkProcessID());
private EdgeValues( TezConfiguration config, ProcessEdge processEdge ) { this.config = config; this.flowElement = processEdge.getFlowElement(); this.ordinals = processEdge.getSourceProvidedOrdinals(); this.resolvedKeyFieldsMap = processEdge.getResolvedKeyFields(); this.resolvedSortFieldsMap = processEdge.getResolvedSortFields(); this.resolvedValueFieldsMap = processEdge.getResolvedValueFields(); }
conf.setOutputKeyComparatorClass( ReverseTupleComparator.class ); Integer ordinal = (Integer) Util.getFirst( processEdge.getSinkExpectedOrdinals() ); addComparators( conf, "cascading.group.comparator", getGroup().getKeySelectors(), (Fields) processEdge.getResolvedKeyFields().get( ordinal ) ); addComparators( conf, "cascading.sort.comparator", getGroup().getSortingSelectors(), (Fields) processEdge.getResolvedSortFields().get( ordinal ) ); if( processEdge != null && ifCoGroupAndKeysHaveCommonTypes( this, processEdge.getFlowElement(), processEdge.getResolvedKeyFields() ) ) conf.set( "cascading.node.ordinals", Util.join( processEdge.getSinkExpectedOrdinals(), "," ) ); addFields( conf, "cascading.node.key.fields", processEdge.getResolvedKeyFields() ); addFields( conf, "cascading.node.sort.fields", processEdge.getResolvedSortFields() ); addFields( conf, "cascading.node.value.fields", processEdge.getResolvedValueFields() );
private void addInputOutputMapping( JobConf conf, FlowNode flowNode ) { FlowNodeGraph flowNodeGraph = getFlowNodeGraph(); Set<ProcessEdge> incomingEdges = flowNodeGraph.incomingEdgesOf( flowNode ); for( ProcessEdge processEdge : incomingEdges ) conf.set( "cascading.node.source." + processEdge.getFlowElementID(), processEdge.getSourceProcessID() ); Set<ProcessEdge> outgoingEdges = flowNodeGraph.outgoingEdgesOf( flowNode ); for( ProcessEdge processEdge : outgoingEdges ) conf.set( "cascading.node.sink." + processEdge.getFlowElementID(), processEdge.getSinkProcessID() ); }
private EdgeProperty createEdgeProperty( TezConfiguration config, ProcessEdge processEdge ) { FlowElement flowElement = processEdge.getFlowElement(); EdgeValues edgeValues = new EdgeValues( new TezConfiguration( config ), processEdge ); edgeValues.keyClassName = KeyTuple.class.getName(); // TEZ_RUNTIME_INTERMEDIATE_OUTPUT_KEY_CLASS edgeValues.valueClassName = ValueTuple.class.getName(); // TEZ_RUNTIME_INTERMEDIATE_OUTPUT_VALUE_CLASS edgeValues.keyComparatorClassName = TupleComparator.class.getName(); edgeValues.keyPartitionerClassName = TuplePartitioner.class.getName(); edgeValues.outputClassName = null; edgeValues.inputClassName = null; edgeValues.movementType = null; edgeValues.sourceType = null; edgeValues.schedulingType = null; if( flowElement instanceof Group ) applyGroup( edgeValues ); else if( ( flowElement instanceof Boundary || flowElement instanceof Merge ) && processEdge.getSinkAnnotations().contains( StreamMode.Accumulated ) ) applyBoundaryMergeAccumulated( edgeValues ); else if( flowElement instanceof Boundary || flowElement instanceof Merge ) applyBoundaryMerge( edgeValues ); else throw new IllegalStateException( "unsupported flow element: " + flowElement.getClass().getCanonicalName() ); applyEdgeAnnotations( processEdge, edgeValues ); return createEdgeProperty( edgeValues ); }
FlowElement flowElement = processEdge.getFlowElement(); List<FlowNode> sourceNodes = nodeGraph.getElementSourceProcesses( flowElement );
private void applyEdgeAnnotations( ProcessEdge processEdge, EdgeValues edgeValues ) { processEdge.addEdgeAnnotation( edgeValues.movementType ); processEdge.addEdgeAnnotation( edgeValues.sourceType ); processEdge.addEdgeAnnotation( edgeValues.schedulingType ); }
protected FlowNodeGraph createFlowNodeGraph( List<ElementGraph> elementGraphs ) { ElementGraph mapElementGraph = elementGraphs.get( 0 ); ElementGraph reduceElementGraph = elementGraphs.size() == 2 ? elementGraphs.get( 1 ) : null; FlowNodeGraph flowNodeGraph = new FlowNodeGraph(); int nodes = elementGraphs.size(); FlowNode mapperNode = new BaseFlowNode( mapElementGraph, String.format( "(1/%s)", nodes ), 0 ); flowNodeGraph.addVertex( mapperNode ); if( nodes == 2 ) { FlowNode reducerNode = new BaseFlowNode( reduceElementGraph, "(2/2)", 1 ); flowNodeGraph.addVertex( reducerNode ); flowNodeGraph.addEdge( mapperNode, reducerNode, new ProcessEdge( mapperNode, reducerNode ) ); } return flowNodeGraph; } }
private EdgeValues( TezConfiguration config, ProcessEdge processEdge ) { this.config = config; this.flowElement = processEdge.getFlowElement(); this.ordinals = processEdge.getSourceProvidedOrdinals(); this.resolvedKeyFieldsMap = processEdge.getResolvedKeyFields(); this.resolvedSortFieldsMap = processEdge.getResolvedSortFields(); this.resolvedValueFieldsMap = processEdge.getResolvedValueFields(); }
conf.setOutputKeyComparatorClass( ReverseTupleComparator.class ); Integer ordinal = (Integer) Util.getFirst( processEdge.getSinkExpectedOrdinals() ); addComparators( conf, "cascading.group.comparator", getGroup().getKeySelectors(), (Fields) processEdge.getResolvedKeyFields().get( ordinal ) ); addComparators( conf, "cascading.sort.comparator", getGroup().getSortingSelectors(), (Fields) processEdge.getResolvedSortFields().get( ordinal ) ); if( processEdge != null && ifCoGroupAndKeysHaveCommonTypes( this, processEdge.getFlowElement(), processEdge.getResolvedKeyFields() ) ) conf.set( "cascading.node.ordinals", Util.join( processEdge.getSinkExpectedOrdinals(), "," ) ); addFields( conf, "cascading.node.key.fields", processEdge.getResolvedKeyFields() ); addFields( conf, "cascading.node.sort.fields", processEdge.getResolvedSortFields() ); addFields( conf, "cascading.node.value.fields", processEdge.getResolvedValueFields() );
private void addInputOutputMapping( JobConf conf, FlowNode flowNode ) { FlowNodeGraph flowNodeGraph = getFlowNodeGraph(); Set<ProcessEdge> incomingEdges = flowNodeGraph.incomingEdgesOf( flowNode ); for( ProcessEdge processEdge : incomingEdges ) conf.set( "cascading.node.source." + processEdge.getFlowElementID(), processEdge.getSourceProcessID() ); Set<ProcessEdge> outgoingEdges = flowNodeGraph.outgoingEdgesOf( flowNode ); for( ProcessEdge processEdge : outgoingEdges ) conf.set( "cascading.node.sink." + processEdge.getFlowElementID(), processEdge.getSinkProcessID() ); }
private EdgeProperty createEdgeProperty( TezConfiguration config, ProcessEdge processEdge ) { FlowElement flowElement = processEdge.getFlowElement(); EdgeValues edgeValues = new EdgeValues( new TezConfiguration( config ), processEdge ); edgeValues.keyClassName = KeyTuple.class.getName(); // TEZ_RUNTIME_INTERMEDIATE_OUTPUT_KEY_CLASS edgeValues.valueClassName = ValueTuple.class.getName(); // TEZ_RUNTIME_INTERMEDIATE_OUTPUT_VALUE_CLASS edgeValues.keyComparatorClassName = TupleComparator.class.getName(); edgeValues.keyPartitionerClassName = TuplePartitioner.class.getName(); edgeValues.outputClassName = null; edgeValues.inputClassName = null; edgeValues.movementType = null; edgeValues.sourceType = null; edgeValues.schedulingType = null; if( flowElement instanceof Group ) applyGroup( edgeValues ); else if( ( flowElement instanceof Boundary || flowElement instanceof Merge ) && processEdge.getSinkAnnotations().contains( StreamMode.Accumulated ) ) applyBoundaryMergeAccumulated( edgeValues ); else if( flowElement instanceof Boundary || flowElement instanceof Merge ) applyBoundaryMerge( edgeValues ); else throw new IllegalStateException( "unsupported flow element: " + flowElement.getClass().getCanonicalName() ); applyEdgeAnnotations( processEdge, edgeValues ); return createEdgeProperty( edgeValues ); }
FlowElement flowElement = processEdge.getFlowElement(); List<FlowNode> sourceNodes = nodeGraph.getElementSourceProcesses( flowElement );
private void applyEdgeAnnotations( ProcessEdge processEdge, EdgeValues edgeValues ) { processEdge.addEdgeAnnotation( edgeValues.movementType ); processEdge.addEdgeAnnotation( edgeValues.sourceType ); processEdge.addEdgeAnnotation( edgeValues.schedulingType ); }
protected FlowNodeGraph createFlowNodeGraph( List<ElementGraph> elementGraphs ) { ElementGraph mapElementGraph = elementGraphs.get( 0 ); ElementGraph reduceElementGraph = elementGraphs.size() == 2 ? elementGraphs.get( 1 ) : null; FlowNodeGraph flowNodeGraph = new FlowNodeGraph(); int nodes = elementGraphs.size(); FlowNode mapperNode = new BaseFlowNode( mapElementGraph, String.format( "(1/%s)", nodes ), 0 ); flowNodeGraph.addVertex( mapperNode ); if( nodes == 2 ) { FlowNode reducerNode = new BaseFlowNode( reduceElementGraph, "(2/2)", 1 ); flowNodeGraph.addVertex( reducerNode ); flowNodeGraph.addEdge( mapperNode, reducerNode, new ProcessEdge( mapperNode, reducerNode ) ); } return flowNodeGraph; } }
graph.addEdge(e.getSourceProcessID(), e.getSinkProcessID());
public void addEdgeAnnotation( Enum annotation ) { if( annotation == null ) return; addEdgeAnnotation( annotation.getDeclaringClass().getName(), annotation.name() ); }
public void bindEdges() { for( Process sinkProcess : vertexSet() ) { for( Process sourceProcess : vertexSet() ) { if( sourceProcess == sinkProcess ) continue; // outer edge sources and sinks to this graph sourceElements.removeAll( sinkProcess.getSinkElements() ); sinkElements.removeAll( sourceProcess.getSourceElements() ); } } for( Process sinkProcess : vertexSet() ) { for( Process sourceProcess : vertexSet() ) { if( sourceProcess == sinkProcess ) continue; for( FlowElement intermediate : sourceProcess.getSinkElements() ) { if( sinkProcess.getSourceElements().contains( intermediate ) ) addEdge( sourceProcess, sinkProcess, new ProcessEdge<>( sourceProcess, intermediate, sinkProcess ) ); } } } // clear tap map caches sourceTaps = null; sinkTaps = null; }