private static InputSpec getInputSpecFromProto(IOSpecProto inputSpecProto) { InputDescriptor inputDescriptor = null; if (inputSpecProto.hasIoDescriptor()) { inputDescriptor = convertInputDescriptorFromProto(inputSpecProto.getIoDescriptor()); } InputSpec inputSpec = new InputSpec(inputSpecProto.getConnectedVertexName(), inputDescriptor, inputSpecProto.getPhysicalEdgeCount()); return inputSpec; }
private static IOSpecProto convertInputSpecToProto(InputSpec inputSpec) { IOSpecProto.Builder builder = IOSpecProto.newBuilder(); if (inputSpec.getSourceVertexName() != null) { builder.setConnectedVertexName(inputSpec.getSourceVertexName()); } if (inputSpec.getInputDescriptor() != null) { builder.setIoDescriptor(convertToProto(inputSpec.getInputDescriptor())); } builder.setPhysicalEdgeCount(inputSpec.getPhysicalEdgeCount()); return builder.build(); }
private List<String> getSourceInterestList(List<InputSpec> inputSpecList) { List<String> sourcesOfInterest = Collections.emptyList(); if (inputSpecList != null) { boolean alreadyFound = false; for (InputSpec inputSpec : inputSpecList) { if (LlapTezUtils.isSourceOfInterest(inputSpec.getInputDescriptor().getClassName())) { if (!alreadyFound) { alreadyFound = true; sourcesOfInterest = new LinkedList<>(); } sourcesOfInterest.add(inputSpec.getSourceVertexName()); } } } return sourcesOfInterest; }
private LogicalInput createInput(InputSpec inputSpec, InputContext inputContext) throws TezException { InputDescriptor inputDesc = inputSpec.getInputDescriptor(); Input input = ReflectionUtils.createClazzInstance(inputDesc.getClassName(), new Class[]{InputContext.class, Integer.TYPE}, new Object[]{inputContext, inputSpec.getPhysicalEdgeCount()}); if (!(input instanceof LogicalInput)) { throw new TezUncheckedException(inputDesc.getClass().getName() + " is not a sub-type of LogicalInput." + " Only LogicalInput sub-types supported by LogicalIOProcessor."); } return (LogicalInput) input; }
inputSpecList = new ArrayList<InputSpec>(numInputSpecs); for (int i = 0; i < numInputSpecs; i++) { InputSpec inputSpec = new InputSpec(); inputSpec.readFields(in); inputSpecList.add(inputSpec);
@Override protected Void callInternal() throws Exception { String oldThreadName = Thread.currentThread().getName(); try { Thread.currentThread().setName(oldThreadName + " Initialize: {" + inputSpec.getSourceVertexName() + "}"); return _callInternal(); } finally { Thread.currentThread().setName(oldThreadName); } }
sb.append(", inputSpecList=["); for (InputSpec i : inputSpecList) { sb.append("{" + i.toString() + "}, ");
private List<String> getSourceInterestList(List<InputSpec> inputSpecList) { List<String> sourcesOfInterest = Collections.emptyList(); if (inputSpecList != null) { boolean alreadyFound = false; for (InputSpec inputSpec : inputSpecList) { if (LlapTezUtils.isSourceOfInterest(inputSpec.getInputDescriptor().getClassName())) { if (!alreadyFound) { alreadyFound = true; sourcesOfInterest = new LinkedList<>(); } sourcesOfInterest.add(inputSpec.getSourceVertexName()); } } } return sourcesOfInterest; }
protected Void _callInternal() throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("Initializing Input using InputSpec: " + inputSpec); } String edgeName = inputSpec.getSourceVertexName(); InputContext inputContext = createInputContext(inputsMap, inputSpec, inputIndex); LogicalInput input = createInput(inputSpec, inputContext); inputsMap.put(edgeName, input); inputContextMap.put(edgeName, inputContext); List<Event> events = ((InputFrameworkInterface)input).initialize(); sendTaskGeneratedEvents(events, EventProducerConsumerType.INPUT, inputContext.getTaskVertexName(), inputContext.getSourceVertexName(), taskSpec.getTaskAttemptID()); initializedInputs.put(edgeName, input); if (LOG.isDebugEnabled()) { LOG.debug("Initialized Input with src edge: " + edgeName); } initializedInputs.put(edgeName, input); return null; } }
private void verifyInputSpecAndProto(InputSpec inputSpec, IOSpecProto inputSpecProto) { assertEquals(inputSpec.getPhysicalEdgeCount(), inputSpecProto.getPhysicalEdgeCount()); assertEquals(inputSpec.getSourceVertexName(), inputSpecProto.getConnectedVertexName()); assertEquals(inputSpec.getInputDescriptor().getClassName(), inputSpecProto.getIoDescriptor().getClassName()); assertEquals(inputSpec.getInputDescriptor().getUserPayload().getPayload(), inputSpecProto.getIoDescriptor().getUserPayload().getUserPayload().asReadOnlyByteBuffer()); }
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
private InputContext createInputContext(Map<String, LogicalInput> inputMap, InputSpec inputSpec, int inputIndex) { InputContext inputContext = new TezInputContextImpl(tezConf, localDirs, appAttemptNumber, tezUmbilical, taskSpec.getDAGName(), taskSpec.getVertexName(), inputSpec.getSourceVertexName(), taskSpec.getVertexParallelism(), taskSpec.getTaskAttemptID(), inputIndex, inputSpec.getInputDescriptor().getUserPayload(), this, serviceConsumerMetadata, envMap, initialMemoryDistributor, inputSpec.getInputDescriptor(), inputMap, inputReadyTracker, objectRegistry, ExecutionContext, memAvailable, sharedExecutor); return inputContext; }
String srcVertexName = inputSpec.getSourceVertexName(); initializedInputs.remove(srcVertexName); List<Event> closeInputEvents = ((InputFrameworkInterface)inputsMap.get(srcVertexName)).close();
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
int numAutoStarts = 0; for (InputSpec inputSpec : inputSpecs) { if (groupInputs.contains(inputSpec.getSourceVertexName())) { LOG.info("Ignoring " + inputSpec.getSourceVertexName() + " for start, since it will be controlled via it's Group"); continue; if (!inputAlreadyStarted(taskSpec.getVertexName(), inputSpec.getSourceVertexName())) { startedInputsMap.put(taskSpec.getVertexName(), inputSpec.getSourceVertexName()); numAutoStarts++; this.initializerCompletionService.submit(new StartInputCallable(inputsMap.get(inputSpec .getSourceVertexName()), inputSpec.getSourceVertexName())); LOG.info("Input: " + inputSpec.getSourceVertexName() + " being auto started by the framework. Subsequent instances will not be auto-started"); if (!groupInputs.contains(inputSpec.getSourceVertexName())) { LogicalInput input = inputsMap.get(inputSpec.getSourceVertexName()); runInputMap.put(inputSpec.getSourceVertexName(), input);
UserPayload input1Payload = UserPayload.create(input1Bb); InputDescriptor id1 = InputDescriptor.create("input1ClassName").setUserPayload(input1Payload); InputSpec inputSpec1 = new InputSpec("sourceVertexName1", id1, 33); InputSpec inputSpec2 = new InputSpec("sourceVertexName2", id1, 44); List<InputSpec> inputSpecList = Lists.newArrayList(inputSpec1, inputSpec2);
for (Entry<String, RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> rootInputDescriptorEntry : rootInputDescriptors.entrySet()) { inputSpecList.add(new InputSpec(rootInputDescriptorEntry.getKey(), rootInputDescriptorEntry.getValue().getIODescriptor(), rootInputSpecs.get( rootInputDescriptorEntry.getKey()).getNumPhysicalInputsForWorkUnit(taskIndex)));
public InputSpec getDestinationSpec(int destinationTaskIndex) throws AMUserCodeException { Preconditions.checkState(edgeManager != null, "Edge Manager must be initialized by this time"); try { int physicalInputCount = edgeManager.getNumDestinationTaskPhysicalInputs(destinationTaskIndex); Preconditions.checkArgument(physicalInputCount >= 0, "PhysicalInputCount should not be negative, " + "physicalInputCount=" + physicalInputCount); return new InputSpec(sourceVertex.getName(), edgeProperty.getEdgeDestination(), physicalInputCount); } catch (Exception e) { throw new AMUserCodeException(Source.EdgeManager, "Fail to getDestinationSpec, destinationTaskIndex=" + destinationTaskIndex +", " + getEdgeInfo(), e); } }