private DAG createDAG( FlowProcess flowProcess, TezConfiguration initializedConfig ) { FlowNodeGraph nodeGraph = getFlowNodeGraph(); Map vertexMap = new HashMap<>(); DAG dag = DAG.create( getStepDisplayName( initializedConfig.getInt( "cascading.display.id.truncate", Util.ID_LENGTH ) ) ); dag.addTaskLocalFiles( allLocalResources ); Iterator iterator = nodeGraph.getOrderedTopologicalIterator(); // ordering of nodes for consistent remote debugging while( iterator.hasNext() ) { FlowNode flowNode = iterator.next(); Vertex vertex = createVertex( flowProcess, initializedConfig, flowNode ); dag.addVertex( vertex ); vertexMap.put( flowNode, vertex ); } LinkedList processedEdges = new LinkedList<>(); for( ProcessEdge processEdge : nodeGraph.edgeSet() ) { if( processedEdges.contains( processEdge ) ) continue; FlowNode edgeTargetFlowNode = nodeGraph.getEdgeTarget( processEdge ); FlowElement flowElement = processEdge.getFlowElement(); List sourceNodes = nodeGraph.getElementSourceProcesses( flowElement ); EdgeProperty edgeProperty = createEdgeProperty( initializedConfig, processEdge ); Vertex targetVertex = vertexMap.get( edgeTargetFlowNode ); if( sourceNodes.size() == 1 || flowElement instanceof CoGroup || flowElement instanceof Boundary ) // todo: create group vertices around incoming ordinal { FlowNode edgeSourceFlowNode = nodeGraph.getEdgeSource( processEdge ); Vertex sourceVertex = vertexMap.get( edgeSourceFlowNode ); LOG.debug( "adding edge between: {} and {}", sourceVertex, targetVertex ); dag.addEdge( Edge.create( sourceVertex, targetVertex, edgeProperty ) ); } else if( flowElement instanceof GroupBy || flowElement instanceof Merge ) // merge - source nodes > 1 { List sourceVerticesIDs = new ArrayList<>(); List sourceVertices = new ArrayList<>(); for( FlowNode edgeSourceFlowNode : sourceNodes ) { sourceVerticesIDs.add( edgeSourceFlowNode.getID() ); sourceVertices.add( vertexMap.get( edgeSourceFlowNode ) ); processedEdges.add( nodeGraph.getEdge( edgeSourceFlowNode, edgeTargetFlowNode ) ); } VertexGroup vertexGroup = dag.createVertexGroup( edgeTargetFlowNode.getID(), sourceVertices.toArray( new Vertex[ sourceVertices.size() ] ) ); String inputClassName = flowElement instanceof Group ? OrderedGroupedMergedKVInput.class.getName() : ConcatenatedMergedKeyValueInput.class.getName(); InputDescriptor inputDescriptor = InputDescriptor.create( inputClassName ).setUserPayload( edgeProperty.getEdgeDestination().getUserPayload() ); String type = ( (Splice) flowElement ).isMerge() ? "merged" : "grouped"; LOG.info( "adding {} edge between: {} and {}", type, Util.join( sourceVerticesIDs, "," ), targetVertex.getName() ); dag.addEdge( GroupInputEdge.create( vertexGroup, targetVertex, edgeProperty, inputDescriptor ) ); } else { throw new UnsupportedOperationException( "can't make edge for: " + flowElement ); } } return dag; }

How to use getElementSourceProcessesmethodin cascading.flow.planner.process.FlowNodeGraph

Best Java code snippets using cascading.flow.planner.process.FlowNodeGraph.getElementSourceProcesses (Showing top 2 results out of 315)

How to use
getElementSourceProcesses
method
in
cascading.flow.planner.process.FlowNodeGraph