private static OutputSpec getOutputSpecFromProto(IOSpecProto outputSpecProto) { OutputDescriptor outputDescriptor = null; if (outputSpecProto.hasIoDescriptor()) { outputDescriptor = convertOutputDescriptorFromProto(outputSpecProto.getIoDescriptor()); } OutputSpec outputSpec = new OutputSpec(outputSpecProto.getConnectedVertexName(), outputDescriptor, outputSpecProto.getPhysicalEdgeCount()); return outputSpec; }
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
OutputDescriptor od1 = OutputDescriptor.create("output1ClassName").setUserPayload(output1Payload); OutputSpec outputSpec1 = new OutputSpec("destVertexName1", od1, 55); OutputSpec outputSpec2 = new OutputSpec("destVertexName2", od1, 66); List<OutputSpec> outputSpecList = Lists.newArrayList(outputSpec1, outputSpec2);
public OutputSpec getSourceSpec(int sourceTaskIndex) throws AMUserCodeException { Preconditions.checkState(edgeManager != null, "Edge Manager must be initialized by this time"); try { int physicalOutputCount = edgeManager.getNumSourceTaskPhysicalOutputs( sourceTaskIndex); Preconditions.checkArgument(physicalOutputCount >= 0, "PhysicalOutputCount should not be negative," + "physicalOutputCount=" + physicalOutputCount); return new OutputSpec(destinationVertex.getName(), edgeProperty.getEdgeSource(), physicalOutputCount); } catch (Exception e) { throw new AMUserCodeException(Source.EdgeManager, "Fail to getSourceSpec, sourceTaskIndex=" + sourceTaskIndex + ", " + getEdgeInfo(), e); } }
outputSpecList = new ArrayList<OutputSpec>(numOutputSpecs); for (int i = 0; i < numOutputSpecs; i++) { OutputSpec outputSpec = new OutputSpec(); outputSpec.readFields(in); outputSpecList.add(outputSpec);
@Override public void setAdditionalOutputs(List<RootInputLeafOutputProto> outputs) { LOG.info("Setting " + outputs.size() + " additional outputs for vertex " + this.logIdentifier); this.additionalOutputs = Maps.newHashMapWithExpectedSize(outputs.size()); this.outputCommitters = Maps.newHashMapWithExpectedSize(outputs.size()); for (RootInputLeafOutputProto output : outputs) { addIO(output.getName()); OutputDescriptor od = DagTypeConverters .convertOutputDescriptorFromDAGPlan(output.getIODescriptor()); this.additionalOutputs .put( output.getName(), new RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>( output.getName(), od, output.hasControllerDescriptor() ? DagTypeConverters .convertOutputCommitterDescriptorFromDAGPlan(output .getControllerDescriptor()) : null)); OutputSpec outputSpec = new OutputSpec(output.getName(), od, 0); additionalOutputSpecs.add(outputSpec); } }