public void setIsGuaranteed(boolean isGuaranteed) { this.isGuaranteed = isGuaranteed; if (amNodeInfo != null) { amNodeInfo.updateTaskAttempt(taskSpec.getTaskAttemptID(), isGuaranteed); } if (wmCounters != null) { wmCounters.changeGuaranteed(isGuaranteed); } }
if (!killInvoked.getAndSet(true)) { synchronized (this) { TezTaskAttemptID ta = taskSpec.getTaskAttemptID(); LOG.info("Kill task requested for id={}, taskRunnerSetup={}", ta, taskRunner != null); shouldRunTask = false; if (taskRunner != null) { killtimerWatch.start(); LOG.info("Issuing kill to task {}", taskSpec.getTaskAttemptID()); boolean killed = taskRunner.killTask(); taskSpec.getTaskAttemptID()); taskSpec.getTaskAttemptID());
/** * Inform the AM that this task has been killed. */ public void reportTaskKilled() { killedTaskHandler .taskKilled(request.getAmHost(), request.getAmPort(), vertex.getTokenIdentifier(), jobToken, fragmentInfo.getQueryInfo().getQueryIdentifier(), taskSpec.getTaskAttemptID()); }
super.registerRunningTaskAttempt(containerId, taskSpec, additionalResources, credentials, credentialsChanged, priority); int dagId = taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId(); if (currentQueryIdentifierProto == null || (dagId != currentQueryIdentifierProto.getDagIndex())) { ", while trying to launch task: " + taskSpec.getTaskAttemptID()); entityTracker.registerTaskAttempt(containerId, taskSpec.getTaskAttemptID(), host, port); nodesForQuery.add(nodeId); fragmentRuntimeInfo = sourceStateTracker.getFragmentRuntimeInfo( taskSpec.getVertexName(), taskSpec.getTaskAttemptID().getTaskID().getId(), priority); } catch (Exception e) { LOG.error( "Error while trying to get runtimeFragmentInfo for fragmentId={}, containerId={}, currentQI={}, currentQueryId={}", taskSpec.getTaskAttemptID(), containerId, currentQueryIdentifierProto, currentHiveQueryId, e); if (e instanceof RuntimeException) { getContext().taskStartedRemotely(taskSpec.getTaskAttemptID(), containerId); communicator.sendSubmitWork(requestProto, host, port, new LlapProtocolClientProxy.ExecuteRequestCallback<SubmitWorkResponseProto>() {
this.threadName = Thread.currentThread().getName(); if (LOG.isDebugEnabled()) { LOG.debug("canFinish: " + taskSpec.getTaskAttemptID() + ": " + canFinish()); TezTaskAttemptID ta = taskSpec.getTaskAttemptID(); this.amReporter.unregisterTask(request.getAmHost(), request.getAmPort(), fragmentInfo.getQueryInfo().getQueryIdentifier(), ta); String fragmentId = LlapTezUtils.stripAttemptPrefix(taskSpec.getTaskAttemptID().toString()); taskReporter = new LlapTaskReporter( completionListener, LOG.info("Not starting task {} since it was killed earlier", taskSpec.getTaskAttemptID()); return new TaskRunner2Result(EndReason.KILL_REQUESTED, null, null, false); if (LOG.isDebugEnabled()) { LOG.debug( "canFinish post completion: " + taskSpec.getTaskAttemptID() + ": " + canFinish());
private SubmitWorkRequestProto constructSubmitWorkRequest(ContainerId containerId, TaskSpec taskSpec, FragmentRuntimeInfo fragmentRuntimeInfo, String hiveQueryId) throws IOException { SubmitWorkRequestProto.Builder builder = SubmitWorkRequestProto.newBuilder(); builder.setFragmentNumber(taskSpec.getTaskAttemptID().getTaskID().getId()); builder.setAttemptNumber(taskSpec.getTaskAttemptID().getId()); builder.setContainerIdString(containerId.toString()); builder.setAmHost(getAmHostString()); builder.setAmPort(getAddress().getPort()); Preconditions.checkState(currentQueryIdentifierProto.getDagIndex() == taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId()); builder.setCredentialsBinary( getCredentials(getContext().getCurrentDagInfo().getCredentials())); builder.setWorkSpec(VertexOrBinary.newBuilder().setVertex(Converters.constructSignableVertexSpec( taskSpec, currentQueryIdentifierProto, getTokenIdentifier(), user, hiveQueryId)).build()); // Don't call builder.setWorkSpecSignature() - Tez doesn't sign fragments builder.setFragmentRuntimeInfo(fragmentRuntimeInfo); if (scheduler != null) { // May be null in tests // TODO: see javadoc builder.setIsGuaranteed(scheduler.isInitialGuaranteed(taskSpec.getTaskAttemptID())); } return builder.build(); }
String user, String hiveQueryIdString) { TezTaskAttemptID tId = taskSpec.getTaskAttemptID();
assertEquals(tezTaskAttemptId, taskSpec.getTaskAttemptID()); assertEquals("fakeProcessorName", taskSpec.getProcessorDescriptor().getClassName()); byte[] serialized = new byte[taskSpec.getProcessorDescriptor().getUserPayload().getPayload().remaining()];
public TezTaskAttemptID getTaskAttemptID() { return taskSpec.getTaskAttemptID(); }
@Override public void registerContainerEnd(ContainerId containerId, ContainerEndReason endReason, String diagnostics) { ContainerInfo containerInfo = registeredContainers.remove(containerId); if (containerInfo != null) { synchronized(containerInfo) { if (containerInfo.taskSpec != null && containerInfo.taskSpec.getTaskAttemptID() != null) { attemptToContainerMap.remove(containerInfo.taskSpec.getTaskAttemptID()); } } } }
@Override public synchronized void handleEvents(Collection<TezEvent> events) { if (events == null || events.isEmpty()) { return; } eventCounter.addAndGet(events.size()); if (LOG.isDebugEnabled()) { LOG.debug("Received events to be processed by task" + ", taskAttemptId=" + taskSpec.getTaskAttemptID() + ", eventCount=" + events.size() + ", newEventCounter=" + eventCounter.get()); } eventsToBeProcessed.addAll(events); }
public void setIsGuaranteed(boolean isGuaranteed) { this.isGuaranteed = isGuaranteed; if (amNodeInfo != null) { amNodeInfo.updateTaskAttempt(taskSpec.getTaskAttemptID(), isGuaranteed); } if (wmCounters != null) { wmCounters.changeGuaranteed(isGuaranteed); } }
protected Void _callInternal() throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("Initializing Input using InputSpec: " + inputSpec); } String edgeName = inputSpec.getSourceVertexName(); InputContext inputContext = createInputContext(inputsMap, inputSpec, inputIndex); LogicalInput input = createInput(inputSpec, inputContext); inputsMap.put(edgeName, input); inputContextMap.put(edgeName, inputContext); List<Event> events = ((InputFrameworkInterface)input).initialize(); sendTaskGeneratedEvents(events, EventProducerConsumerType.INPUT, inputContext.getTaskVertexName(), inputContext.getSourceVertexName(), taskSpec.getTaskAttemptID()); initializedInputs.put(edgeName, input); if (LOG.isDebugEnabled()) { LOG.debug("Initialized Input with src edge: " + edgeName); } initializedInputs.put(edgeName, input); return null; } }
protected Void _callInternal() throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("Initializing Output using OutputSpec: " + outputSpec); } String edgeName = outputSpec.getDestinationVertexName(); OutputContext outputContext = createOutputContext(outputSpec, outputIndex); LogicalOutput output = createOutput(outputSpec, outputContext); outputsMap.put(edgeName, output); outputContextMap.put(edgeName, outputContext); List<Event> events = ((OutputFrameworkInterface)output).initialize(); sendTaskGeneratedEvents(events, EventProducerConsumerType.OUTPUT, outputContext.getTaskVertexName(), outputContext.getDestinationVertexName(), taskSpec.getTaskAttemptID()); initializedOutputs.put(edgeName, output); if (LOG.isDebugEnabled()) { LOG.debug("Initialized Output with dest edge: " + edgeName); } initializedOutputs.put(edgeName, output); return null; } }
@Override protected ContainerTask callInternal() throws Exception { ContainerTask containerTask = null; LOG.info("Attempting to fetch new task for container {}", containerContext.getContainerIdentifier()); containerTask = umbilical.getTask(containerContext); long getTaskPollStartTime = System.currentTimeMillis(); nextGetTaskPrintTime = getTaskPollStartTime + LOG_INTERVAL; for (int idle = 1; containerTask == null; idle++) { long sleepTimeMilliSecs = Math.min(idle * 10, getTaskMaxSleepTime); maybeLogSleepMessage(sleepTimeMilliSecs); TimeUnit.MILLISECONDS.sleep(sleepTimeMilliSecs); containerTask = umbilical.getTask(containerContext); } LOG.info("Got TaskUpdate for containerId= " + containerContext.getContainerIdentifier() + ": " + (System.currentTimeMillis() - getTaskPollStartTime) + " ms after starting to poll." + " TaskInfo: shouldDie: " + containerTask.shouldDie() + (containerTask.shouldDie() == true ? "" : ", currentTaskAttemptId: " + containerTask.getTaskSpec().getTaskAttemptID())); return containerTask; }
private ProcessorContext createProcessorContext() { ProcessorContext processorContext = new TezProcessorContextImpl(tezConf, localDirs, appAttemptNumber, tezUmbilical, taskSpec.getDAGName(), taskSpec.getVertexName(), taskSpec.getVertexParallelism(), taskSpec.getTaskAttemptID(), processorDescriptor.getUserPayload(), this, serviceConsumerMetadata, envMap, initialMemoryDistributor, processorDescriptor, inputReadyTracker, objectRegistry, ExecutionContext, memAvailable, sharedExecutor); return processorContext; }
/** * Inform the AM that this task has been killed. */ public void reportTaskKilled() { killedTaskHandler .taskKilled(request.getAmHost(), request.getAmPort(), vertex.getTokenIdentifier(), jobToken, fragmentInfo.getQueryInfo().getQueryIdentifier(), taskSpec.getTaskAttemptID()); }
/** * Cleans entries from the object registry, and resets the startedInputsMap if required * * @param containerTask * the new task specification. Must be a valid task */ private void cleanupOnTaskChanged(ContainerTask containerTask) { Preconditions.checkState(!containerTask.shouldDie()); Preconditions.checkState(containerTask.getTaskSpec() != null); TezVertexID newVertexID = containerTask.getTaskSpec().getTaskAttemptID().getTaskID() .getVertexID(); if (lastVertexID != null) { if (!lastVertexID.equals(newVertexID)) { objectRegistry.clearCache(ObjectRegistryImpl.ObjectLifeCycle.VERTEX); } if (!lastVertexID.getDAGId().equals(newVertexID.getDAGId())) { objectRegistry.clearCache(ObjectRegistryImpl.ObjectLifeCycle.DAG); startedInputsMap = HashMultimap.create(); } } lastVertexID = newVertexID; }
private OutputContext createOutputContext(OutputSpec outputSpec, int outputIndex) { OutputContext outputContext = new TezOutputContextImpl(tezConf, localDirs, appAttemptNumber, tezUmbilical, taskSpec.getDAGName(), taskSpec.getVertexName(), outputSpec.getDestinationVertexName(), taskSpec.getVertexParallelism(), taskSpec.getTaskAttemptID(), outputIndex, outputSpec.getOutputDescriptor().getUserPayload(), this, serviceConsumerMetadata, envMap, initialMemoryDistributor, outputSpec.getOutputDescriptor(), objectRegistry, ExecutionContext, memAvailable, sharedExecutor); return outputContext; }
private InputContext createInputContext(Map<String, LogicalInput> inputMap, InputSpec inputSpec, int inputIndex) { InputContext inputContext = new TezInputContextImpl(tezConf, localDirs, appAttemptNumber, tezUmbilical, taskSpec.getDAGName(), taskSpec.getVertexName(), inputSpec.getSourceVertexName(), taskSpec.getVertexParallelism(), taskSpec.getTaskAttemptID(), inputIndex, inputSpec.getInputDescriptor().getUserPayload(), this, serviceConsumerMetadata, envMap, initialMemoryDistributor, inputSpec.getInputDescriptor(), inputMap, inputReadyTracker, objectRegistry, ExecutionContext, memAvailable, sharedExecutor); return inputContext; }