private List<String> getSourceInterestList(List<InputSpec> inputSpecList) { List<String> sourcesOfInterest = Collections.emptyList(); if (inputSpecList != null) { boolean alreadyFound = false; for (InputSpec inputSpec : inputSpecList) { if (LlapTezUtils.isSourceOfInterest(inputSpec.getInputDescriptor().getClassName())) { if (!alreadyFound) { alreadyFound = true; sourcesOfInterest = new LinkedList<>(); } sourcesOfInterest.add(inputSpec.getSourceVertexName()); } } } return sourcesOfInterest; }
private static IOSpecProto convertInputSpecToProto(InputSpec inputSpec) { IOSpecProto.Builder builder = IOSpecProto.newBuilder(); if (inputSpec.getSourceVertexName() != null) { builder.setConnectedVertexName(inputSpec.getSourceVertexName()); } if (inputSpec.getInputDescriptor() != null) { builder.setIoDescriptor(convertToProto(inputSpec.getInputDescriptor())); } builder.setPhysicalEdgeCount(inputSpec.getPhysicalEdgeCount()); return builder.build(); }
private void verifyInputSpecAndProto(InputSpec inputSpec, IOSpecProto inputSpecProto) { assertEquals(inputSpec.getPhysicalEdgeCount(), inputSpecProto.getPhysicalEdgeCount()); assertEquals(inputSpec.getSourceVertexName(), inputSpecProto.getConnectedVertexName()); assertEquals(inputSpec.getInputDescriptor().getClassName(), inputSpecProto.getIoDescriptor().getClassName()); assertEquals(inputSpec.getInputDescriptor().getUserPayload().getPayload(), inputSpecProto.getIoDescriptor().getUserPayload().getUserPayload().asReadOnlyByteBuffer()); }
@Override protected Void callInternal() throws Exception { String oldThreadName = Thread.currentThread().getName(); try { Thread.currentThread().setName(oldThreadName + " Initialize: {" + inputSpec.getSourceVertexName() + "}"); return _callInternal(); } finally { Thread.currentThread().setName(oldThreadName); } }
private List<String> getSourceInterestList(List<InputSpec> inputSpecList) { List<String> sourcesOfInterest = Collections.emptyList(); if (inputSpecList != null) { boolean alreadyFound = false; for (InputSpec inputSpec : inputSpecList) { if (LlapTezUtils.isSourceOfInterest(inputSpec.getInputDescriptor().getClassName())) { if (!alreadyFound) { alreadyFound = true; sourcesOfInterest = new LinkedList<>(); } sourcesOfInterest.add(inputSpec.getSourceVertexName()); } } } return sourcesOfInterest; }
protected Void _callInternal() throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("Initializing Input using InputSpec: " + inputSpec); } String edgeName = inputSpec.getSourceVertexName(); InputContext inputContext = createInputContext(inputsMap, inputSpec, inputIndex); LogicalInput input = createInput(inputSpec, inputContext); inputsMap.put(edgeName, input); inputContextMap.put(edgeName, inputContext); List<Event> events = ((InputFrameworkInterface)input).initialize(); sendTaskGeneratedEvents(events, EventProducerConsumerType.INPUT, inputContext.getTaskVertexName(), inputContext.getSourceVertexName(), taskSpec.getTaskAttemptID()); initializedInputs.put(edgeName, input); if (LOG.isDebugEnabled()) { LOG.debug("Initialized Input with src edge: " + edgeName); } initializedInputs.put(edgeName, input); return null; } }
String srcVertexName = inputSpec.getSourceVertexName(); initializedInputs.remove(srcVertexName); List<Event> closeInputEvents = ((InputFrameworkInterface)inputsMap.get(srcVertexName)).close();
int numAutoStarts = 0; for (InputSpec inputSpec : inputSpecs) { if (groupInputs.contains(inputSpec.getSourceVertexName())) { LOG.info("Ignoring " + inputSpec.getSourceVertexName() + " for start, since it will be controlled via it's Group"); continue; if (!inputAlreadyStarted(taskSpec.getVertexName(), inputSpec.getSourceVertexName())) { startedInputsMap.put(taskSpec.getVertexName(), inputSpec.getSourceVertexName()); numAutoStarts++; this.initializerCompletionService.submit(new StartInputCallable(inputsMap.get(inputSpec .getSourceVertexName()), inputSpec.getSourceVertexName())); LOG.info("Input: " + inputSpec.getSourceVertexName() + " being auto started by the framework. Subsequent instances will not be auto-started"); if (!groupInputs.contains(inputSpec.getSourceVertexName())) { LogicalInput input = inputsMap.get(inputSpec.getSourceVertexName()); runInputMap.put(inputSpec.getSourceVertexName(), input);
private InputContext createInputContext(Map<String, LogicalInput> inputMap, InputSpec inputSpec, int inputIndex) { InputContext inputContext = new TezInputContextImpl(tezConf, localDirs, appAttemptNumber, tezUmbilical, taskSpec.getDAGName(), taskSpec.getVertexName(), inputSpec.getSourceVertexName(), taskSpec.getVertexParallelism(), taskSpec.getTaskAttemptID(), inputIndex, inputSpec.getInputDescriptor().getUserPayload(), this, serviceConsumerMetadata, envMap, initialMemoryDistributor, inputSpec.getInputDescriptor(), inputMap, inputReadyTracker, objectRegistry, ExecutionContext, memAvailable, sharedExecutor); return inputContext; }