public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
String user, String hiveQueryIdString) { TezTaskAttemptID tId = taskSpec.getTaskAttemptID(); builder.setHiveQueryId(hiveQueryIdString); builder.setVertexIndex(tId.getTaskID().getVertexID().getId()); builder.setDagName(taskSpec.getDAGName()); builder.setVertexName(taskSpec.getVertexName()); builder.setVertexParallelism(taskSpec.getVertexParallelism()); builder.setTokenIdentifier(tokenIdentifier); builder.setUser(user); if (taskSpec.getProcessorDescriptor() != null) { builder.setProcessorDescriptor( convertToProto(taskSpec.getProcessorDescriptor())); if (taskSpec.getInputs() != null && !taskSpec.getInputs().isEmpty()) { for (InputSpec inputSpec : taskSpec.getInputs()) { builder.addInputSpecs(convertInputSpecToProto(inputSpec)); if (taskSpec.getOutputs() != null && !taskSpec.getOutputs().isEmpty()) { for (OutputSpec outputSpec : taskSpec.getOutputs()) { builder.addOutputSpecs(convertOutputSpecToProto(outputSpec)); if (taskSpec.getGroupInputs() != null && !taskSpec.getGroupInputs().isEmpty()) { for (GroupInputSpec groupInputSpec : taskSpec.getGroupInputs()) { builder.addGroupedInputSpecs(convertGroupInputSpecToProto(groupInputSpec));
super.registerRunningTaskAttempt(containerId, taskSpec, additionalResources, credentials, credentialsChanged, priority); int dagId = taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId(); if (currentQueryIdentifierProto == null || (dagId != currentQueryIdentifierProto.getDagIndex())) { ", while trying to launch task: " + taskSpec.getTaskAttemptID()); entityTracker.registerTaskAttempt(containerId, taskSpec.getTaskAttemptID(), host, port); nodesForQuery.add(nodeId); sourceStateTracker.registerTaskForStateUpdates(host, port, taskSpec.getInputs()); FragmentRuntimeInfo fragmentRuntimeInfo; try { fragmentRuntimeInfo = sourceStateTracker.getFragmentRuntimeInfo( taskSpec.getVertexName(), taskSpec.getTaskAttemptID().getTaskID().getId(), priority); } catch (Exception e) { LOG.error( "Error while trying to get runtimeFragmentInfo for fragmentId={}, containerId={}, currentQI={}, currentQueryId={}", taskSpec.getTaskAttemptID(), containerId, currentQueryIdentifierProto, currentHiveQueryId, e); if (e instanceof RuntimeException) { getContext().taskStartedRemotely(taskSpec.getTaskAttemptID(), containerId); communicator.sendSubmitWork(requestProto, host, port, new LlapProtocolClientProxy.ExecuteRequestCallback<SubmitWorkResponseProto>() {
assertEquals("dagName", taskSpec.getDAGName()); assertEquals("vertexName", taskSpec.getVertexName()); assertEquals(tezTaskAttemptId, taskSpec.getTaskAttemptID()); assertEquals("fakeProcessorName", taskSpec.getProcessorDescriptor().getClassName()); byte[] serialized = new byte[taskSpec.getProcessorDescriptor().getUserPayload().getPayload().remaining()]; taskSpec.getProcessorDescriptor().getUserPayload().getPayload().get(serialized); assertArrayEquals(procBb.array(), serialized); assertEquals(2, taskSpec.getInputs().size()); assertEquals(2, taskSpec.getOutputs().size()); verifyInputSpecAndProto(taskSpec.getInputs().get(0), vertexProto.getInputSpecs(0)); verifyInputSpecAndProto(taskSpec.getInputs().get(1), vertexProto.getInputSpecs(1)); verifyOutputSpecAndProto(taskSpec.getOutputs().get(0), vertexProto.getOutputSpecs(0)); verifyOutputSpecAndProto(taskSpec.getOutputs().get(1), vertexProto.getOutputSpecs(1));
TaskAttemptImpl createAttempt(int attemptNumber, TezTaskAttemptID schedulingCausalTA) { TezTaskAttemptID attemptId = TezBuilderUtils.newTaskAttemptId(taskId, attemptNumber); TaskSpec taskSpec = new TaskSpec(attemptId, baseTaskSpec.getDAGName(), baseTaskSpec.getVertexName(), baseTaskSpec.getVertexParallelism(), baseTaskSpec.getProcessorDescriptor(), baseTaskSpec.getInputs(), baseTaskSpec.getOutputs(), baseTaskSpec.getGroupInputs(), baseTaskSpec.getTaskConf()); return new TaskAttemptImpl(attemptId, eventHandler, taskCommunicatorManagerInterface, conf, clock, taskHeartbeatHandler, appContext, (failedAttempts > 0), taskResource, containerContext, leafVertex, getVertex(), locationHint, taskSpec, schedulingCausalTA); }
private ProcessorContext createProcessorContext() { ProcessorContext processorContext = new TezProcessorContextImpl(tezConf, localDirs, appAttemptNumber, tezUmbilical, taskSpec.getDAGName(), taskSpec.getVertexName(), taskSpec.getVertexParallelism(), taskSpec.getTaskAttemptID(), processorDescriptor.getUserPayload(), this, serviceConsumerMetadata, envMap, initialMemoryDistributor, processorDescriptor, inputReadyTracker, objectRegistry, ExecutionContext, memAvailable, sharedExecutor); return processorContext; }
public void setIsGuaranteed(boolean isGuaranteed) { this.isGuaranteed = isGuaranteed; if (amNodeInfo != null) { amNodeInfo.updateTaskAttempt(taskSpec.getTaskAttemptID(), isGuaranteed); } if (wmCounters != null) { wmCounters.changeGuaranteed(isGuaranteed); } }
LOG.info("Initializing LogicalIOProcessorRuntimeTask with TaskSpec: " + taskSpec); int numInputs = taskSpec.getInputs().size(); int numOutputs = taskSpec.getOutputs().size(); this.localDirs = localDirs; this.inputSpecs = taskSpec.getInputs(); this.inputsMap = new ConcurrentHashMap<String, LogicalInput>(numInputs); this.inputContextMap = new ConcurrentHashMap<String, InputContext>(numInputs); this.outputSpecs = taskSpec.getOutputs(); this.outputsMap = new ConcurrentHashMap<String, LogicalOutput>(numOutputs); this.outputContextMap = new ConcurrentHashMap<String, OutputContext>(numOutputs); this.initializedOutputs = new ConcurrentHashMap<String, LogicalOutput>(); this.processorDescriptor = taskSpec.getProcessorDescriptor(); this.serviceConsumerMetadata = serviceConsumerMetadata; this.envMap = envMap; this.initializerCompletionService = new ExecutorCompletionService<Void>( this.initializerExecutor); this.groupInputSpecs = taskSpec.getGroupInputs(); initialMemoryDistributor = new MemoryDistributor(numInputs, numOutputs, tezConf); this.startedInputsMap = startedInputsMap;
List<Event> closeInputEvents = ((InputFrameworkInterface)inputsMap.get(srcVertexName)).close(); sendTaskGeneratedEvents(closeInputEvents, EventProducerConsumerType.INPUT, taskSpec.getVertexName(), srcVertexName, taskSpec.getTaskAttemptID()); List<Event> closeOutputEvents = ((LogicalOutputFrameworkInterface)outputsMap.get(destVertexName)).close(); sendTaskGeneratedEvents(closeOutputEvents, EventProducerConsumerType.OUTPUT, taskSpec.getVertexName(), destVertexName, taskSpec.getTaskAttemptID()); System.err.println(timeStamp + " Completed running task attempt: " + taskSpec.getTaskAttemptID().toString()); System.out.println(timeStamp + " Completed running task attempt: " + taskSpec.getTaskAttemptID().toString());
for (InputSpec inputSpec : taskSpec.getInputs()) { this.initializerCompletionService.submit( new InitializeInputCallable(inputSpec, inputIndex++)); for (OutputSpec outputSpec : taskSpec.getOutputs()) { this.initializerCompletionService.submit( new InitializeOutputCallable(outputSpec, outputIndex++)); continue; if (!inputAlreadyStarted(taskSpec.getVertexName(), inputSpec.getSourceVertexName())) { startedInputsMap.put(taskSpec.getVertexName(), inputSpec.getSourceVertexName()); numAutoStarts++; this.initializerCompletionService.submit(new StartInputCallable(inputsMap.get(inputSpec if (!inputAlreadyStarted(taskSpec.getVertexName(), group.getGroupName())) { numAutoStarts++; this.initializerCompletionService.submit(new StartInputCallable(groupInputsMap.get(group
private String extractQueryId(TaskSpec taskSpec) throws IOException { UserPayload processorPayload = taskSpec.getProcessorDescriptor().getUserPayload(); Configuration conf = TezUtils.createConfFromUserPayload(processorPayload); return HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYID); }
System.currentTimeMillis(), taskSpec.getVertexParallelism(), signedSvs.message, signedSvs.signature, umbilicalToken); byte[] submitWorkBytes = SubmitWorkInfo.toBytes(submitWorkInfo);
@Override public void readFields(DataInput in) throws IOException { shouldDie = in.readBoolean(); boolean taskComing = in.readBoolean(); if (taskComing) { taskSpec = new TaskSpec(); taskSpec.readFields(in); } int numAdditionalResources = in.readInt(); additionalResources = Maps.newHashMap(); if (numAdditionalResources != -1) { for (int i = 0 ; i < numAdditionalResources ; i++) { String resourceName = in.readUTF(); TezLocalResource localResource = new TezLocalResource(); localResource.readFields(in); additionalResources.put(resourceName, localResource); } } credentialsChanged = in.readBoolean(); if (credentialsChanged) { boolean hasCredentials = in.readBoolean(); if (hasCredentials) { credentials = new Credentials(); credentials.readFields(in); } } }
public String getVertexName() { return taskSpec.getVertexName(); }
private OutputContext createOutputContext(OutputSpec outputSpec, int outputIndex) { OutputContext outputContext = new TezOutputContextImpl(tezConf, localDirs, appAttemptNumber, tezUmbilical, taskSpec.getDAGName(), taskSpec.getVertexName(), outputSpec.getDestinationVertexName(), taskSpec.getVertexParallelism(), taskSpec.getTaskAttemptID(), outputIndex, outputSpec.getOutputDescriptor().getUserPayload(), this, serviceConsumerMetadata, envMap, initialMemoryDistributor, outputSpec.getOutputDescriptor(), objectRegistry, ExecutionContext, memAvailable, sharedExecutor); return outputContext; }
if (!killInvoked.getAndSet(true)) { synchronized (this) { TezTaskAttemptID ta = taskSpec.getTaskAttemptID(); LOG.info("Kill task requested for id={}, taskRunnerSetup={}", ta, taskRunner != null); shouldRunTask = false; if (taskRunner != null) { killtimerWatch.start(); LOG.info("Issuing kill to task {}", taskSpec.getTaskAttemptID()); boolean killed = taskRunner.killTask(); taskSpec.getTaskAttemptID()); taskSpec.getTaskAttemptID());
super.registerRunningTaskAttempt(containerId, taskSpec, additionalResources, credentials, credentialsChanged, priority); int dagId = taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId(); if (currentQueryIdentifierProto == null || (dagId != currentQueryIdentifierProto.getDagIndex())) { ", while trying to launch task: " + taskSpec.getTaskAttemptID()); entityTracker.registerTaskAttempt(containerId, taskSpec.getTaskAttemptID(), host, port); nodesForQuery.add(nodeId); sourceStateTracker.registerTaskForStateUpdates(host, port, taskSpec.getInputs()); FragmentRuntimeInfo fragmentRuntimeInfo; try { fragmentRuntimeInfo = sourceStateTracker.getFragmentRuntimeInfo( taskSpec.getVertexName(), taskSpec.getTaskAttemptID().getTaskID().getId(), priority); } catch (Exception e) { LOG.error( "Error while trying to get runtimeFragmentInfo for fragmentId={}, containerId={}, currentQI={}, currentQueryId={}", taskSpec.getTaskAttemptID(), containerId, currentQueryIdentifierProto, currentHiveQueryId, e); if (e instanceof RuntimeException) { getContext().taskStartedRemotely(taskSpec.getTaskAttemptID(), containerId); communicator.sendSubmitWork(requestProto, host, port, new LlapProtocolClientProxy.ExecuteRequestCallback<SubmitWorkResponseProto>() {
private boolean handleEvent(TezEvent e) { if (LOG.isDebugEnabled()) { LOG.debug("Handling TezEvent in task" + ", taskAttemptId=" + taskSpec.getTaskAttemptID() + ", eventType=" + e.getEventType() + ", eventSourceInfo=" + e.getSourceInfo() EventMetaData sourceInfo = new EventMetaData( e.getDestinationInfo().getEventGenerator(), taskSpec.getVertexName(), e.getDestinationInfo().getEdgeVertexName(), getTaskAttemptID()); setFrameworkCounters();
private String extractQueryId(TaskSpec taskSpec) throws IOException { UserPayload processorPayload = taskSpec.getProcessorDescriptor().getUserPayload(); Configuration conf = TezUtils.createConfFromUserPayload(processorPayload); return HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYID); }
System.currentTimeMillis(), taskSpec.getVertexParallelism(), signedSvs.message, signedSvs.signature); byte[] submitWorkBytes = SubmitWorkInfo.toBytes(submitWorkInfo);