private static OutputSpec getOutputSpecFromProto(IOSpecProto outputSpecProto) { OutputDescriptor outputDescriptor = null; if (outputSpecProto.hasIoDescriptor()) { outputDescriptor = convertOutputDescriptorFromProto(outputSpecProto.getIoDescriptor()); } OutputSpec outputSpec = new OutputSpec(outputSpecProto.getConnectedVertexName(), outputDescriptor, outputSpecProto.getPhysicalEdgeCount()); return outputSpec; }
public static IOSpecProto convertOutputSpecToProto(OutputSpec outputSpec) { IOSpecProto.Builder builder = IOSpecProto.newBuilder(); if (outputSpec.getDestinationVertexName() != null) { builder.setConnectedVertexName(outputSpec.getDestinationVertexName()); } if (outputSpec.getOutputDescriptor() != null) { builder.setIoDescriptor(convertToProto(outputSpec.getOutputDescriptor())); } builder.setPhysicalEdgeCount(outputSpec.getPhysicalEdgeCount()); return builder.build(); }
outputSpecList = new ArrayList<OutputSpec>(numOutputSpecs); for (int i = 0; i < numOutputSpecs; i++) { OutputSpec outputSpec = new OutputSpec(); outputSpec.readFields(in); outputSpecList.add(outputSpec);
private LogicalOutput createOutput(OutputSpec outputSpec, OutputContext outputContext) throws TezException { OutputDescriptor outputDesc = outputSpec.getOutputDescriptor(); Output output = ReflectionUtils.createClazzInstance(outputDesc.getClassName(), new Class[]{OutputContext.class, Integer.TYPE}, new Object[]{outputContext, outputSpec.getPhysicalEdgeCount()}); if (!(output instanceof LogicalOutput)) { throw new TezUncheckedException(output.getClass().getName() + " is not a sub-type of LogicalOutput." + " Only LogicalOutput sub-types supported by LogicalIOProcessor."); } return (LogicalOutput) output; }
private OutputContext createOutputContext(OutputSpec outputSpec, int outputIndex) { OutputContext outputContext = new TezOutputContextImpl(tezConf, localDirs, appAttemptNumber, tezUmbilical, taskSpec.getDAGName(), taskSpec.getVertexName(), outputSpec.getDestinationVertexName(), taskSpec.getVertexParallelism(), taskSpec.getTaskAttemptID(), outputIndex, outputSpec.getOutputDescriptor().getUserPayload(), this, serviceConsumerMetadata, envMap, initialMemoryDistributor, outputSpec.getOutputDescriptor(), objectRegistry, ExecutionContext, memAvailable, sharedExecutor); return outputContext; }
@Override protected Void callInternal() throws Exception { String oldThreadName = Thread.currentThread().getName(); try { Thread.currentThread().setName(oldThreadName + " Initialize: {" + outputSpec.getDestinationVertexName() + "}"); return _callInternal(); } finally { Thread.currentThread().setName(oldThreadName); } }
sb.append("{" + i.toString() + "}, ");
protected Void _callInternal() throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("Initializing Output using OutputSpec: " + outputSpec); } String edgeName = outputSpec.getDestinationVertexName(); OutputContext outputContext = createOutputContext(outputSpec, outputIndex); LogicalOutput output = createOutput(outputSpec, outputContext); outputsMap.put(edgeName, output); outputContextMap.put(edgeName, outputContext); List<Event> events = ((OutputFrameworkInterface)output).initialize(); sendTaskGeneratedEvents(events, EventProducerConsumerType.OUTPUT, outputContext.getTaskVertexName(), outputContext.getDestinationVertexName(), taskSpec.getTaskAttemptID()); initializedOutputs.put(edgeName, output); if (LOG.isDebugEnabled()) { LOG.debug("Initialized Output with dest edge: " + edgeName); } initializedOutputs.put(edgeName, output); return null; } }
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
private void verifyOutputSpecAndProto(OutputSpec outputSpec, IOSpecProto outputSpecProto) { assertEquals(outputSpec.getPhysicalEdgeCount(), outputSpecProto.getPhysicalEdgeCount()); assertEquals(outputSpec.getDestinationVertexName(), outputSpecProto.getConnectedVertexName()); assertEquals(outputSpec.getOutputDescriptor().getClassName(), outputSpecProto.getIoDescriptor().getClassName()); assertEquals(outputSpec.getOutputDescriptor().getUserPayload().getPayload(), outputSpecProto.getIoDescriptor().getUserPayload().getUserPayload().asReadOnlyByteBuffer()); } }
String destVertexName = outputSpec.getDestinationVertexName(); initializedOutputs.remove(destVertexName); List<Event> closeOutputEvents = ((LogicalOutputFrameworkInterface)outputsMap.get(destVertexName)).close();
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
LogicalOutput output = outputsMap.get(outputSpec.getDestinationVertexName()); String outputName = outputSpec.getDestinationVertexName(); runOutputMap.put(outputName, output);
OutputDescriptor od1 = OutputDescriptor.create("output1ClassName").setUserPayload(output1Payload); OutputSpec outputSpec1 = new OutputSpec("destVertexName1", od1, 55); OutputSpec outputSpec2 = new OutputSpec("destVertexName2", od1, 66); List<OutputSpec> outputSpecList = Lists.newArrayList(outputSpec1, outputSpec2);
public OutputSpec getSourceSpec(int sourceTaskIndex) throws AMUserCodeException { Preconditions.checkState(edgeManager != null, "Edge Manager must be initialized by this time"); try { int physicalOutputCount = edgeManager.getNumSourceTaskPhysicalOutputs( sourceTaskIndex); Preconditions.checkArgument(physicalOutputCount >= 0, "PhysicalOutputCount should not be negative," + "physicalOutputCount=" + physicalOutputCount); return new OutputSpec(destinationVertex.getName(), edgeProperty.getEdgeSource(), physicalOutputCount); } catch (Exception e) { throw new AMUserCodeException(Source.EdgeManager, "Fail to getSourceSpec, sourceTaskIndex=" + sourceTaskIndex + ", " + getEdgeInfo(), e); } }
@Override public void setAdditionalOutputs(List<RootInputLeafOutputProto> outputs) { LOG.info("Setting " + outputs.size() + " additional outputs for vertex " + this.logIdentifier); this.additionalOutputs = Maps.newHashMapWithExpectedSize(outputs.size()); this.outputCommitters = Maps.newHashMapWithExpectedSize(outputs.size()); for (RootInputLeafOutputProto output : outputs) { addIO(output.getName()); OutputDescriptor od = DagTypeConverters .convertOutputDescriptorFromDAGPlan(output.getIODescriptor()); this.additionalOutputs .put( output.getName(), new RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>( output.getName(), od, output.hasControllerDescriptor() ? DagTypeConverters .convertOutputCommitterDescriptorFromDAGPlan(output .getControllerDescriptor()) : null)); OutputSpec outputSpec = new OutputSpec(output.getName(), od, 0); additionalOutputSpecs.add(outputSpec); } }