Tabnine Logo
FlowNodeGraph
Code IndexAdd Tabnine to your IDE (free)

How to use
FlowNodeGraph
in
cascading.flow.planner.process

Best Java code snippets using cascading.flow.planner.process.FlowNodeGraph (Showing top 20 results out of 315)

origin: dataArtisans/cascading-flink

private void printFlowStep() {
  Iterator<FlowNode> iterator = getFlowNodeGraph().getTopologicalIterator();
  LOG.info("Step Cnt: {} ", getFlowNodeGraph().vertexSet().size());
  LOG.info("Edge Cnt: {} ", getFlowNodeGraph().edgeSet().size());
  LOG.info("Src Set: {} ", getFlowNodeGraph().getSourceElements());
  LOG.info("Snk Set: {} ", getFlowNodeGraph().getSinkElements());
  LOG.info("##############");
  while(iterator.hasNext()) {
    FlowNode next = iterator.next();
    LOG.info("Node cnt: {} ", next.getElementGraph().vertexSet().size());
    LOG.info("Edge cnt: {} ", next.getElementGraph().edgeSet().size());
    LOG.info("Nodes: {} ", next.getElementGraph().vertexSet());
    LOG.info("-----------");
  }
}
origin: cwensel/cascading

public FlowNodeGraph( FlowNodeFactory flowNodeFactory, FlowElementGraph flowElementGraph, List<? extends ElementGraph> nodeSubGraphs, Map<ElementGraph, List<? extends ElementGraph>> pipelineSubGraphsMap )
 {
 buildGraph( flowNodeFactory, flowElementGraph, nodeSubGraphs, pipelineSubGraphsMap );
 // consistently sets ordinal of node based on topological dependencies and tie breaking by the given Comparator
 Iterator<FlowNode> iterator = getOrderedTopologicalIterator();
 int ordinal = 0;
 int size = vertexSet().size();
 while( iterator.hasNext() )
  {
  BaseFlowNode next = (BaseFlowNode) iterator.next();
  next.setOrdinal( ordinal );
  next.setName( flowNodeFactory.makeFlowNodeName( next, size, ordinal ) );
  ordinal++;
  }
 }
origin: cwensel/cascading

private ElementGraph createStepElementGraph( FlowNodeGraph flowNodeGraph )
 {
 return ElementGraphs.asElementDirectedGraph( flowNodeGraph.getElementGraphs() ).bindExtents();
 }
origin: cwensel/cascading

protected void buildGraph( FlowNodeFactory flowNodeFactory, FlowElementGraph flowElementGraph, List<? extends ElementGraph> nodeSubGraphs, Map<ElementGraph, List<? extends ElementGraph>> pipelineSubGraphsMap )
 {
 if( pipelineSubGraphsMap == null )
  pipelineSubGraphsMap = Collections.emptyMap();
 for( ElementGraph nodeSubGraph : nodeSubGraphs )
  {
  List<? extends ElementGraph> pipelineGraphs = pipelineSubGraphsMap.get( nodeSubGraph );
  FlowNode flowNode = flowNodeFactory.createFlowNode( flowElementGraph, nodeSubGraph, pipelineGraphs );
  addVertex( flowNode );
  }
 bindEdges();
 }
origin: cwensel/cascading

protected FlowNodeGraph createFlowNodeGraph( List<ElementGraph> elementGraphs )
 {
 ElementGraph mapElementGraph = elementGraphs.get( 0 );
 ElementGraph reduceElementGraph = elementGraphs.size() == 2 ? elementGraphs.get( 1 ) : null;
 FlowNodeGraph flowNodeGraph = new FlowNodeGraph();
 int nodes = elementGraphs.size();
 FlowNode mapperNode = new BaseFlowNode( mapElementGraph, String.format( "(1/%s)", nodes ), 0 );
 flowNodeGraph.addVertex( mapperNode );
 if( nodes == 2 )
  {
  FlowNode reducerNode = new BaseFlowNode( reduceElementGraph, "(2/2)", 1 );
  flowNodeGraph.addVertex( reducerNode );
  flowNodeGraph.addEdge( mapperNode, reducerNode, new ProcessEdge( mapperNode, reducerNode ) );
  }
 return flowNodeGraph;
 }
}
origin: cwensel/cascading

Iterator<FlowNode> iterator = nodeGraph.getOrderedTopologicalIterator(); // ordering of nodes for consistent remote debugging
for( ProcessEdge processEdge : nodeGraph.edgeSet() )
 FlowNode edgeTargetFlowNode = nodeGraph.getEdgeTarget( processEdge );
 List<FlowNode> sourceNodes = nodeGraph.getElementSourceProcesses( flowElement );
  FlowNode edgeSourceFlowNode = nodeGraph.getEdgeSource( processEdge );
  Vertex sourceVertex = vertexMap.get( edgeSourceFlowNode );
   processedEdges.add( nodeGraph.getEdge( edgeSourceFlowNode, edgeTargetFlowNode ) );
origin: cwensel/cascading

ProcessEdge processEdge = Util.getFirst( getFlowNodeGraph().edgeSet() );
Iterator<FlowNode> iterator = getFlowNodeGraph().getTopologicalIterator();
origin: cwensel/cascading

@Override
public int getNumFlowNodes()
 {
 return flowNodeGraph.vertexSet().size();
 }
origin: cwensel/cascading

for( FlowNode current : step.getFlowNodeGraph().vertexSet() )
 if( step.getFlowNodeGraph().inDegreeOf( current ) == 0 )
origin: cwensel/cascading

flowNodeGraph.writeDOTNested( stepNodeElementGraphName, stepSubGraph );
flowNodeGraph.writeDOT( stepNodeGraphName );
Iterator<FlowNode> iterator = flowNodeGraph.getOrderedTopologicalIterator();
origin: cwensel/cascading

Iterator<FlowNode> iterator = flowStep.getFlowNodeGraph().getOrderedTopologicalIterator();
origin: cwensel/cascading

protected FlowNodeGraph createFlowNodeGraph( FlowStepFactory flowStepFactory, FlowElementGraph flowElementGraph, Map<ElementGraph, List<? extends ElementGraph>> pipelineSubGraphsMap, List<? extends ElementGraph> nodeSubGraphs )
 {
 return new FlowNodeGraph( flowStepFactory.getFlowNodeFactory(), flowElementGraph, nodeSubGraphs, pipelineSubGraphsMap );
 }
}
origin: dataArtisans/cascading-flink

Iterator<FlowNode> iterator = flowNodeGraph.getTopologicalIterator();
origin: cascading/cascading-hadoop2-tez

Iterator<FlowNode> iterator = nodeGraph.getOrderedTopologicalIterator(); // ordering of nodes for consistent remote debugging
for( ProcessEdge processEdge : nodeGraph.edgeSet() )
 FlowNode edgeTargetFlowNode = nodeGraph.getEdgeTarget( processEdge );
 List<FlowNode> sourceNodes = nodeGraph.getElementSourceProcesses( flowElement );
  FlowNode edgeSourceFlowNode = nodeGraph.getEdgeSource( processEdge );
  Vertex sourceVertex = vertexMap.get( edgeSourceFlowNode );
   processedEdges.add( nodeGraph.getEdge( edgeSourceFlowNode, edgeTargetFlowNode ) );
origin: cascading/cascading-hadoop2-mr1

protected FlowNodeGraph createFlowNodeGraph( List<ElementGraph> elementGraphs )
 {
 ElementGraph mapElementGraph = elementGraphs.get( 0 );
 ElementGraph reduceElementGraph = elementGraphs.size() == 2 ? elementGraphs.get( 1 ) : null;
 FlowNodeGraph flowNodeGraph = new FlowNodeGraph();
 int nodes = elementGraphs.size();
 FlowNode mapperNode = new BaseFlowNode( mapElementGraph, String.format( "(1/%s)", nodes ), 0 );
 flowNodeGraph.addVertex( mapperNode );
 if( nodes == 2 )
  {
  FlowNode reducerNode = new BaseFlowNode( reduceElementGraph, "(2/2)", 1 );
  flowNodeGraph.addVertex( reducerNode );
  flowNodeGraph.addEdge( mapperNode, reducerNode, new ProcessEdge( mapperNode, reducerNode ) );
  }
 return flowNodeGraph;
 }
}
origin: cascading/cascading-hadoop2-mr1

ProcessEdge processEdge = Util.getFirst( getFlowNodeGraph().edgeSet() );
Iterator<FlowNode> iterator = getFlowNodeGraph().getTopologicalIterator();
origin: cwensel/cascading

public Set<FlowElement> getFlowElementsFor( Enum annotation )
 {
 Set<FlowElement> results = createIdentitySet();
 for( FlowNode flowNode : vertexSet() )
  results.addAll( flowNode.getFlowElementsFor( annotation ) );
 return results;
 }
origin: cascading/cascading-hadoop2-mr1

for( FlowNode current : step.getFlowNodeGraph().vertexSet() )
 if( step.getFlowNodeGraph().inDegreeOf( current ) == 0 )
origin: cascading/cascading-hadoop2-tez-stats

Iterator<FlowNode> iterator = flowStep.getFlowNodeGraph().getOrderedTopologicalIterator();
origin: cwensel/cascading

public LocalStepRunner( FlowProcess<Properties> flowProcess, LocalFlowStep step )
 {
 this.currentProcess = flowProcess;
 this.flowNode = Util.getFirst( step.getFlowNodeGraph().vertexSet() );
 this.streamGraph = new LocalStepStreamGraph( this.currentProcess, step, flowNode );
 this.heads = streamGraph.getHeads();
 }
cascading.flow.planner.processFlowNodeGraph

Most used methods

  • edgeSet
  • getOrderedTopologicalIterator
  • vertexSet
  • getElementGraphs
  • <init>
  • addVertex
  • getTopologicalIterator
  • addEdge
  • getEdge
  • getEdgeSource
  • getEdgeTarget
  • getElementSourceProcesses
  • getEdgeTarget,
  • getElementSourceProcesses,
  • inDegreeOf,
  • incomingEdgesOf,
  • outgoingEdgesOf,
  • bindEdges,
  • buildGraph,
  • getAnnotations,
  • getFlowElementsFor,
  • getSinkElements

Popular in Java

  • Reading from database using SQL prepared statement
  • getResourceAsStream (ClassLoader)
  • getExternalFilesDir (Context)
  • getSharedPreferences (Context)
  • UnknownHostException (java.net)
    Thrown when a hostname can not be resolved.
  • ByteBuffer (java.nio)
    A buffer for bytes. A byte buffer can be created in either one of the following ways: * #allocate
  • Enumeration (java.util)
    A legacy iteration interface.New code should use Iterator instead. Iterator replaces the enumeration
  • Reference (javax.naming)
  • Options (org.apache.commons.cli)
    Main entry-point into the library. Options represents a collection of Option objects, which describ
  • Project (org.apache.tools.ant)
    Central representation of an Ant project. This class defines an Ant project with all of its targets,
  • CodeWhisperer alternatives
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now