private EdgeProperty createEdgeProperty( TezConfiguration config, ProcessEdge processEdge ) { FlowElement flowElement = processEdge.getFlowElement(); EdgeValues edgeValues = new EdgeValues( new TezConfiguration( config ), processEdge ); edgeValues.keyClassName = KeyTuple.class.getName(); // TEZ_RUNTIME_INTERMEDIATE_OUTPUT_KEY_CLASS edgeValues.valueClassName = ValueTuple.class.getName(); // TEZ_RUNTIME_INTERMEDIATE_OUTPUT_VALUE_CLASS edgeValues.keyComparatorClassName = TupleComparator.class.getName(); edgeValues.keyPartitionerClassName = TuplePartitioner.class.getName(); edgeValues.outputClassName = null; edgeValues.inputClassName = null; edgeValues.movementType = null; edgeValues.sourceType = null; edgeValues.schedulingType = null; if( flowElement instanceof Group ) applyGroup( edgeValues ); else if( ( flowElement instanceof Boundary || flowElement instanceof Merge ) && processEdge.getSinkAnnotations().contains( StreamMode.Accumulated ) ) applyBoundaryMergeAccumulated( edgeValues ); else if( flowElement instanceof Boundary || flowElement instanceof Merge ) applyBoundaryMerge( edgeValues ); else throw new IllegalStateException( "unsupported flow element: " + flowElement.getClass().getCanonicalName() ); applyEdgeAnnotations( processEdge, edgeValues ); return createEdgeProperty( edgeValues ); }
private EdgeProperty createEdgeProperty( TezConfiguration config, ProcessEdge processEdge ) { FlowElement flowElement = processEdge.getFlowElement(); EdgeValues edgeValues = new EdgeValues( new TezConfiguration( config ), processEdge ); edgeValues.keyClassName = KeyTuple.class.getName(); // TEZ_RUNTIME_INTERMEDIATE_OUTPUT_KEY_CLASS edgeValues.valueClassName = ValueTuple.class.getName(); // TEZ_RUNTIME_INTERMEDIATE_OUTPUT_VALUE_CLASS edgeValues.keyComparatorClassName = TupleComparator.class.getName(); edgeValues.keyPartitionerClassName = TuplePartitioner.class.getName(); edgeValues.outputClassName = null; edgeValues.inputClassName = null; edgeValues.movementType = null; edgeValues.sourceType = null; edgeValues.schedulingType = null; if( flowElement instanceof Group ) applyGroup( edgeValues ); else if( ( flowElement instanceof Boundary || flowElement instanceof Merge ) && processEdge.getSinkAnnotations().contains( StreamMode.Accumulated ) ) applyBoundaryMergeAccumulated( edgeValues ); else if( flowElement instanceof Boundary || flowElement instanceof Merge ) applyBoundaryMerge( edgeValues ); else throw new IllegalStateException( "unsupported flow element: " + flowElement.getClass().getCanonicalName() ); applyEdgeAnnotations( processEdge, edgeValues ); return createEdgeProperty( edgeValues ); }
private EdgeValues( TezConfiguration config, ProcessEdge processEdge ) { this.config = config; this.flowElement = processEdge.getFlowElement(); this.ordinals = processEdge.getSourceProvidedOrdinals(); this.resolvedKeyFieldsMap = processEdge.getResolvedKeyFields(); this.resolvedSortFieldsMap = processEdge.getResolvedSortFields(); this.resolvedValueFieldsMap = processEdge.getResolvedValueFields(); }
private EdgeValues( TezConfiguration config, ProcessEdge processEdge ) { this.config = config; this.flowElement = processEdge.getFlowElement(); this.ordinals = processEdge.getSourceProvidedOrdinals(); this.resolvedKeyFieldsMap = processEdge.getResolvedKeyFields(); this.resolvedSortFieldsMap = processEdge.getResolvedSortFields(); this.resolvedValueFieldsMap = processEdge.getResolvedValueFields(); }
FlowElement flowElement = processEdge.getFlowElement(); List<FlowNode> sourceNodes = nodeGraph.getElementSourceProcesses( flowElement );
FlowElement flowElement = processEdge.getFlowElement(); List<FlowNode> sourceNodes = nodeGraph.getElementSourceProcesses( flowElement );
if( processEdge != null && ifCoGroupAndKeysHaveCommonTypes( this, processEdge.getFlowElement(), processEdge.getResolvedKeyFields() ) )
if( processEdge != null && ifCoGroupAndKeysHaveCommonTypes( this, processEdge.getFlowElement(), processEdge.getResolvedKeyFields() ) )