private static InputSpec getInputSpecFromProto(IOSpecProto inputSpecProto) { InputDescriptor inputDescriptor = null; if (inputSpecProto.hasIoDescriptor()) { inputDescriptor = convertInputDescriptorFromProto(inputSpecProto.getIoDescriptor()); } InputSpec inputSpec = new InputSpec(inputSpecProto.getConnectedVertexName(), inputDescriptor, inputSpecProto.getPhysicalEdgeCount()); return inputSpec; }
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
UserPayload input1Payload = UserPayload.create(input1Bb); InputDescriptor id1 = InputDescriptor.create("input1ClassName").setUserPayload(input1Payload); InputSpec inputSpec1 = new InputSpec("sourceVertexName1", id1, 33); InputSpec inputSpec2 = new InputSpec("sourceVertexName2", id1, 44); List<InputSpec> inputSpecList = Lists.newArrayList(inputSpec1, inputSpec2);
for (Entry<String, RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> rootInputDescriptorEntry : rootInputDescriptors.entrySet()) { inputSpecList.add(new InputSpec(rootInputDescriptorEntry.getKey(), rootInputDescriptorEntry.getValue().getIODescriptor(), rootInputSpecs.get( rootInputDescriptorEntry.getKey()).getNumPhysicalInputsForWorkUnit(taskIndex)));
public InputSpec getDestinationSpec(int destinationTaskIndex) throws AMUserCodeException { Preconditions.checkState(edgeManager != null, "Edge Manager must be initialized by this time"); try { int physicalInputCount = edgeManager.getNumDestinationTaskPhysicalInputs(destinationTaskIndex); Preconditions.checkArgument(physicalInputCount >= 0, "PhysicalInputCount should not be negative, " + "physicalInputCount=" + physicalInputCount); return new InputSpec(sourceVertex.getName(), edgeProperty.getEdgeDestination(), physicalInputCount); } catch (Exception e) { throw new AMUserCodeException(Source.EdgeManager, "Fail to getDestinationSpec, destinationTaskIndex=" + destinationTaskIndex +", " + getEdgeInfo(), e); } }
inputSpecList = new ArrayList<InputSpec>(numInputSpecs); for (int i = 0; i < numInputSpecs; i++) { InputSpec inputSpec = new InputSpec(); inputSpec.readFields(in); inputSpecList.add(inputSpec);