private static OutputDescriptor convertOutputDescriptorFromProto( EntityDescriptorProto proto) { String className = proto.getClassName(); UserPayload payload = convertPayloadFromProto(proto); OutputDescriptor od = OutputDescriptor.create(className); setUserPayload(od, payload); return od; }
OutputDescriptor.create(MROutput.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(conf)), null, null));
OutputDescriptor.create(MROutput.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(conf)), null, null));
UserPayload output1Payload = UserPayload.create(output1Bb); OutputDescriptor od1 = OutputDescriptor.create("output1ClassName").setUserPayload(output1Payload); OutputSpec outputSpec1 = new OutputSpec("destVertexName1", od1, 55); OutputSpec outputSpec2 = new OutputSpec("destVertexName2", od1, 66);
public static OutputDescriptor convertOutputDescriptorFromDAGPlan( TezEntityDescriptorProto proto) { String className = proto.getClassName(); UserPayload payload = convertTezUserPayloadFromDAGPlan(proto); OutputDescriptor od = OutputDescriptor.create(className); setUserPayload(od, payload); return od; }
OutputDescriptor.create(outputClassName).setUserPayload(createUserPayload()), (doCommit ? OutputCommitterDescriptor.create( MROutputCommitter.class.getName()) : null), null);
/** * This is a convenience method for creating an Edge descriptor based on the specified * EdgeManagerDescriptor. * * @param edgeManagerDescriptor the custom edge specification * @return an {@link org.apache.tez.dag.api.EdgeProperty} instance */ public EdgeProperty createDefaultCustomEdgeProperty(EdgeManagerPluginDescriptor edgeManagerDescriptor) { Preconditions.checkNotNull(edgeManagerDescriptor, "EdgeManagerDescriptor cannot be null"); EdgeProperty edgeProperty = EdgeProperty.create(edgeManagerDescriptor, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create(getOutputClassName()).setUserPayload(getOutputPayload()), InputDescriptor.create(getInputClassName()).setUserPayload(getInputPayload())); Utils.setEdgePropertyHistoryText(this, edgeProperty); return edgeProperty; }
/** * This is a convenience method for creating an Edge descriptor based on the specified * EdgeManagerDescriptor. * * @param edgeManagerDescriptor the custom edge specification * @return an {@link org.apache.tez.dag.api.EdgeProperty} instance */ public EdgeProperty createDefaultCustomEdgeProperty(EdgeManagerPluginDescriptor edgeManagerDescriptor) { Preconditions.checkNotNull(edgeManagerDescriptor, "EdgeManagerDescriptor cannot be null"); EdgeProperty edgeProperty = EdgeProperty.create(edgeManagerDescriptor, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create(getOutputClassName()).setUserPayload(getOutputPayload()), InputDescriptor.create(getInputClassName()).setUserPayload(getInputPayload())); Utils.setEdgePropertyHistoryText(this, edgeProperty); return edgeProperty; }
/** * This is a convenience method for creating an Edge descriptor based on the specified * EdgeManagerDescriptor. * * @param edgeManagerDescriptor the custom edge specification * @return an {@link org.apache.tez.dag.api.EdgeProperty} instance */ public EdgeProperty createDefaultCustomEdgeProperty(EdgeManagerPluginDescriptor edgeManagerDescriptor) { Preconditions.checkNotNull(edgeManagerDescriptor, "EdgeManagerDescriptor cannot be null"); EdgeProperty edgeProperty = EdgeProperty.create(edgeManagerDescriptor, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create(getOutputClassName()).setUserPayload(getOutputPayload()), InputDescriptor.create(getInputClassName()).setUserPayload(getInputPayload())); Utils.setEdgePropertyHistoryText(this, edgeProperty); return edgeProperty; }
v2.addDataSource(INPUT, dataSourceDescriptor); OutputDescriptor outputDescriptor = OutputDescriptor.create(FakeOutput.class.getName()); OutputCommitterDescriptor outputCommitterDescriptor = OutputCommitterDescriptor.create(FakeOutputCommitter.class.getName());
DataSinkDescriptor.create(OutputDescriptor.create(MROutput.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf)), OutputCommitterDescriptor.create(MROutputCommitter.class.getName()), null));
OutputDescriptor od = OutputDescriptor.create(MROutputLegacy.class.getName()) .setUserPayload(vertexUserPayload); if (stageConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
OutputDescriptor out = OutputDescriptor.create(edge.outputClassName);
/** * This is a convenience method for the typical usage of this edge, and creates an instance of * {@link org.apache.tez.dag.api.EdgeProperty} which is likely to be used. </p> * If custom edge properties are required, the methods to get the relevant payloads should be * used. </p> * * In this case - DataMovementType.BROADCAST, EdgeProperty.DataSourceType.PERSISTED, * EdgeProperty.SchedulingType.SEQUENTIAL * * @return an {@link org.apache.tez.dag.api.EdgeProperty} instance */ public EdgeProperty createDefaultBroadcastEdgeProperty() { EdgeProperty edgeProperty = EdgeProperty.create(EdgeProperty.DataMovementType.BROADCAST, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create( getOutputClassName()).setUserPayload(getOutputPayload()), InputDescriptor.create( getInputClassName()).setUserPayload(getInputPayload())); Utils.setEdgePropertyHistoryText(this, edgeProperty); return edgeProperty; }
/** * This is a convenience method for the typical usage of this edge, and creates an instance of * {@link org.apache.tez.dag.api.EdgeProperty} which is likely to be used. </p> * * In this case - DataMovementType.SCATTER_GATHER, EdgeProperty.DataSourceType.PERSISTED, * EdgeProperty.SchedulingType.SEQUENTIAL * * @return an {@link org.apache.tez.dag.api.EdgeProperty} instance */ public EdgeProperty createDefaultEdgeProperty() { EdgeProperty edgeProperty = EdgeProperty.create(EdgeProperty.DataMovementType.SCATTER_GATHER, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create( getOutputClassName()).setUserPayload(getOutputPayload()), InputDescriptor.create( getInputClassName()).setUserPayload(getInputPayload())); Utils.setEdgePropertyHistoryText(this, edgeProperty); return edgeProperty; }
/** * This is a convenience method for the typical usage of this edge, and creates an instance of * {@link org.apache.tez.dag.api.EdgeProperty} which is likely to be used. </p> * If custom edge properties are required, the methods to get the relevant payloads should be * used. </p> * * In this case - DataMovementType.SCATTER_GATHER, EdgeProperty.DataSourceType.PERSISTED, * EdgeProperty.SchedulingType.SEQUENTIAL * * @return an {@link org.apache.tez.dag.api.EdgeProperty} instance */ public EdgeProperty createDefaultEdgeProperty() { EdgeProperty edgeProperty = EdgeProperty.create(EdgeProperty.DataMovementType.SCATTER_GATHER, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create( getOutputClassName()).setUserPayload(getOutputPayload()), InputDescriptor.create( getInputClassName()).setUserPayload(getInputPayload())); Utils.setEdgePropertyHistoryText(this, edgeProperty); return edgeProperty; }
/** * This is a convenience method for the typical usage of this edge, and creates an instance of * {@link org.apache.tez.dag.api.EdgeProperty} which is likely to be used. </p> * If custom edge properties are required, the methods to get the relevant payloads should be * used. </p> * * In this case - DataMovementType.ONE_TO_ONE, EdgeProperty.DataSourceType.PERSISTED, * EdgeProperty.SchedulingType.SEQUENTIAL * * @return an {@link org.apache.tez.dag.api.EdgeProperty} instance */ public EdgeProperty createDefaultOneToOneEdgeProperty() { EdgeProperty edgeProperty = EdgeProperty.create(EdgeProperty.DataMovementType.ONE_TO_ONE, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create( getOutputClassName()).setUserPayload(getOutputPayload()), InputDescriptor.create( getInputClassName()).setUserPayload(getInputPayload())); Utils.setEdgePropertyHistoryText(this, edgeProperty); return edgeProperty; }
OutputDescriptor.create(MROutput.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(conf)), null, null));
private EdgeProperty createEdgeProperty( EdgeValues edgeValues ) { TezConfiguration outputConfig = new TezConfiguration( edgeValues.getConfig() ); outputConfig.set( "cascading.node.sink", FlowElements.id( edgeValues.getFlowElement() ) ); outputConfig.set( "cascading.node.ordinals", Util.join( edgeValues.getOrdinals(), "," ) ); addFields( outputConfig, "cascading.node.key.fields", edgeValues.getResolvedKeyFieldsMap() ); addFields( outputConfig, "cascading.node.sort.fields", edgeValues.getResolvedSortFieldsMap() ); addFields( outputConfig, "cascading.node.value.fields", edgeValues.getResolvedValueFieldsMap() ); UserPayload outputPayload = createIntermediatePayloadOutput( outputConfig, edgeValues ); TezConfiguration inputConfig = new TezConfiguration( edgeValues.getConfig() ); inputConfig.set( "cascading.node.source", FlowElements.id( edgeValues.getFlowElement() ) ); inputConfig.set( "cascading.node.ordinals", Util.join( edgeValues.getOrdinals(), "," ) ); addFields( inputConfig, "cascading.node.key.fields", edgeValues.getResolvedKeyFieldsMap() ); addFields( inputConfig, "cascading.node.sort.fields", edgeValues.getResolvedSortFieldsMap() ); addFields( inputConfig, "cascading.node.value.fields", edgeValues.getResolvedValueFieldsMap() ); UserPayload inputPayload = createIntermediatePayloadInput( inputConfig, edgeValues ); return EdgeProperty.create( edgeValues.getMovementType(), edgeValues.getSourceType(), edgeValues.getSchedulingType(), OutputDescriptor.create( edgeValues.getOutputClassName() ).setUserPayload( outputPayload ), InputDescriptor.create( edgeValues.getInputClassName() ).setUserPayload( inputPayload ) ); }
private EdgeProperty createEdgeProperty( EdgeValues edgeValues ) { TezConfiguration outputConfig = new TezConfiguration( edgeValues.getConfig() ); outputConfig.set( "cascading.node.sink", FlowElements.id( edgeValues.getFlowElement() ) ); outputConfig.set( "cascading.node.ordinals", Util.join( edgeValues.getOrdinals(), "," ) ); addFields( outputConfig, "cascading.node.key.fields", edgeValues.getResolvedKeyFieldsMap() ); addFields( outputConfig, "cascading.node.sort.fields", edgeValues.getResolvedSortFieldsMap() ); addFields( outputConfig, "cascading.node.value.fields", edgeValues.getResolvedValueFieldsMap() ); UserPayload outputPayload = createIntermediatePayloadOutput( outputConfig, edgeValues ); TezConfiguration inputConfig = new TezConfiguration( edgeValues.getConfig() ); inputConfig.set( "cascading.node.source", FlowElements.id( edgeValues.getFlowElement() ) ); inputConfig.set( "cascading.node.ordinals", Util.join( edgeValues.getOrdinals(), "," ) ); addFields( inputConfig, "cascading.node.key.fields", edgeValues.getResolvedKeyFieldsMap() ); addFields( inputConfig, "cascading.node.sort.fields", edgeValues.getResolvedSortFieldsMap() ); addFields( inputConfig, "cascading.node.value.fields", edgeValues.getResolvedValueFieldsMap() ); UserPayload inputPayload = createIntermediatePayloadInput( inputConfig, edgeValues ); return EdgeProperty.create( edgeValues.getMovementType(), edgeValues.getSourceType(), edgeValues.getSchedulingType(), OutputDescriptor.create( edgeValues.getOutputClassName() ).setUserPayload( outputPayload ), InputDescriptor.create( edgeValues.getInputClassName() ).setUserPayload( inputPayload ) ); }