public NodeStreamGraph( FlowProcess flowProcess, FlowNode node ) { this.flowProcess = flowProcess; this.node = node; this.elementGraph = node.getElementGraph(); }
public NodeStreamGraph( FlowProcess flowProcess, FlowNode node, FlowElement streamedSource ) { this.flowProcess = flowProcess; this.node = node; this.elementGraph = streamedSource == null ? node.getElementGraph() : node.getPipelineGraphFor( streamedSource ); this.streamedSource = streamedSource; }
private FlowElement getCommonSuccessor(Set<FlowElement> set, FlowNode node) { ElementGraph graph = node.getElementGraph(); FlowElement successor = null; for(FlowElement e : set) { List<FlowElement> successors = graph.successorListOf(e); if(successors.size() > 1) { return null; } else { if(successor == null) { successor = successors.get(0); } else if(successor != successors.get(0)){ return null; } } } return successor; }
private Set<FlowElement> getInnerElements(FlowNode node) { Set<FlowElement> inner = new HashSet<>(node.getElementGraph().vertexSet()); inner.removeAll(getSources(node)); inner.removeAll(getSinks(node)); Set<FlowElement> toRemove = new HashSet<>(); for(FlowElement e : inner) { if(e instanceof Extent) { toRemove.add(e); } } inner.removeAll(toRemove); return inner; }
@Override public int compare( FlowNode lhs, FlowNode rhs ) { // larger graph first int lhsSize = lhs.getElementGraph().vertexSet().size(); int rhsSize = rhs.getElementGraph().vertexSet().size(); int result = ( lhsSize < rhsSize ) ? -1 : ( ( lhsSize == rhsSize ) ? 0 : 1 ); if( result != 0 ) return result; // more inputs second lhsSize = lhs.getSourceElements().size(); rhsSize = rhs.getSourceElements().size(); return ( lhsSize < rhsSize ) ? -1 : ( ( lhsSize == rhsSize ) ? 0 : 1 ); } }
private void initFromNodeConfigDef( FlowNode flowNode, Configuration conf ) { initConfFromNodeConfigDef( flowNode.getElementGraph(), new ConfigurationSetter( conf ) ); }
private void initFromNodeConfigDef( FlowNode flowNode, Configuration conf ) { initConfFromNodeConfigDef( flowNode.getElementGraph(), new ConfigurationSetter( conf ) ); }
private void printFlowStep() { Iterator<FlowNode> iterator = getFlowNodeGraph().getTopologicalIterator(); LOG.info("Step Cnt: {} ", getFlowNodeGraph().vertexSet().size()); LOG.info("Edge Cnt: {} ", getFlowNodeGraph().edgeSet().size()); LOG.info("Src Set: {} ", getFlowNodeGraph().getSourceElements()); LOG.info("Snk Set: {} ", getFlowNodeGraph().getSinkElements()); LOG.info("##############"); while(iterator.hasNext()) { FlowNode next = iterator.next(); LOG.info("Node cnt: {} ", next.getElementGraph().vertexSet().size()); LOG.info("Edge cnt: {} ", next.getElementGraph().edgeSet().size()); LOG.info("Nodes: {} ", next.getElementGraph().vertexSet()); LOG.info("-----------"); } }
private FlowElement[] getNodeInputsInOrder(FlowNode node, Splice splice) { Map<String, Integer> posMap = splice.getPipePos(); FlowElement[] spliceInputs = new FlowElement[posMap.size()]; ElementGraph eg = node.getElementGraph(); for(FlowElement nodeSource : getSources(node)) { int idx = posMap.get(eg.getEdge(nodeSource, splice).getName()); spliceInputs[idx] = nodeSource; } return spliceInputs; }
public SinkBoundaryInStage(FlowProcess flowProcess, FlowElement flowElement, FlowNode node) { super(flowProcess, flowElement); this.nextStarted = false; Scope inScope = node.getElementGraph().incomingEdgesOf(flowElement).iterator().next(); Fields inFields; if(inScope.isEvery()) { inFields = inScope.getOutGroupingFields(); } else { inFields = inScope.getOutValuesFields(); } this.tupleEntry = new TupleEntry(inFields); }
private Configuration getNodeConfig(FlowNode node) { Configuration nodeConfig = HadoopUtil.copyConfiguration(this.getConfig()); ConfigurationSetter configSetter = new ConfigurationSetter(nodeConfig); this.initConfFromNodeConfigDef(node.getElementGraph(), configSetter); this.initConfFromStepConfigDef(configSetter); nodeConfig.set("cascading.flow.node.num", Integer.toString(node.getOrdinal())); return nodeConfig; }
ElementGraph nodeGraph = flowNode.getElementGraph(); int nodeOrdinal = flowNode.getOrdinal(); String nodeGraphName = String.format( "%s/%04d-%04d-step-node-sub-graph-%s.dot", rootPath, stepOrdinal, nodeOrdinal, canonicalHash( nodeGraph ) );
private void initFromNodeConfigDef( final Properties properties ) { initConfFromNodeConfigDef( Util.getFirst( getFlowNodeGraph().vertexSet() ).getElementGraph(), getSetterFor( properties ) ); }
FlowNode node = iterator.next(); Set<FlowElement> all = node.getElementGraph().vertexSet(); Set<FlowElement> sources = getSources(node); Set<FlowElement> sinks = getSinks(node); throw new RuntimeException("Could not translate this node: "+node.getElementGraph().vertexSet());