private String constructLlapLogUrl(final TezTaskAttemptID attemptID, final String containerIdString, final boolean isDone, final String nmAddress) { String dagId = attemptID.getTaskID().getVertexID().getDAGId().toString(); String filename = JOINER.join(currentHiveQueryId, "-", dagId, ".log", (isDone ? ".done" : ""), "?nm.id=", nmAddress); String url = PATH_JOINER.join(timelineServerUri, "ws", "v1", "applicationhistory", "containers", containerIdString, "logs", filename); return url; }
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
super.registerRunningTaskAttempt(containerId, taskSpec, additionalResources, credentials, credentialsChanged, priority); int dagId = taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId(); if (currentQueryIdentifierProto == null || (dagId != currentQueryIdentifierProto.getDagIndex())) { fragmentRuntimeInfo = sourceStateTracker.getFragmentRuntimeInfo( taskSpec.getVertexName(), taskSpec.getTaskAttemptID().getTaskID().getId(), priority); } catch (Exception e) { LOG.error(
private static TimelineEntity convertTaskStartedEvent(TaskStartedEvent event) { TimelineEntity atsEntity = new TimelineEntity(); atsEntity.setEntityId(event.getTaskID().toString()); atsEntity.setEntityType(EntityTypes.TEZ_TASK_ID.name()); atsEntity.addRelatedEntity(EntityTypes.TEZ_VERTEX_ID.name(), event.getTaskID().getVertexID().toString()); atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID, event.getTaskID().getVertexID().getDAGId().getApplicationId().toString()); atsEntity.addPrimaryFilter(EntityTypes.TEZ_DAG_ID.name(), event.getTaskID().getVertexID().getDAGId().toString()); atsEntity.addPrimaryFilter(EntityTypes.TEZ_VERTEX_ID.name(), event.getTaskID().getVertexID().toString()); TimelineEvent startEvt = new TimelineEvent(); startEvt.setEventType(HistoryEventType.TASK_STARTED.name()); startEvt.setTimestamp(event.getStartTime()); atsEntity.addEvent(startEvt); atsEntity.setStartTime(event.getStartTime()); atsEntity.addOtherInfo(ATSConstants.START_TIME, event.getStartTime()); atsEntity.addOtherInfo(ATSConstants.SCHEDULED_TIME, event.getScheduledTime()); atsEntity.addOtherInfo(ATSConstants.STATUS, event.getState().name()); return atsEntity; }
String vertexId = TezTaskID.fromString(taskInfo.getTaskId()).getVertexID().toString(); VertexInfo vertexInfo = dagInfo.getVertexFromId(vertexId); Preconditions.checkState(vertexInfo != null, "VertexInfo for " + vertexId + " can't be " .getTaskAttemptId()); VertexInfo vertexInfo = dagInfo.getVertexFromId(taskAttemptId.getTaskID() .getVertexID().toString()); Preconditions.checkState(vertexInfo != null, "Vertex " + taskAttemptId.getTaskID() .getVertexID().toString() + " is not present in DAG"); TaskInfo taskInfo = vertexInfo.getTask(taskAttemptId.getTaskID().toString()); attemptInfo.setTaskInfo(taskInfo);
@Override public int getIdentifier() { return taskId.getId(); }
TezTaskID taskId = TezTaskID.getInstance(vertexId, task.getTaskIndex()); TaskSpec baseTaskSpec = createRemoteTaskSpec(taskId.getId()); boolean fromRecovery = recoveryData == null ? false : recoveryData.getTaskRecoveryData(taskId) != null; eventHandler.handle(new TaskEventScheduleTask(taskId, baseTaskSpec,
case Constants.TEZ_TASK_ID: String taskName = entity; TezTaskID tezTaskID = TezTaskID.fromString(taskName); if (!tezDAGID.equals(tezTaskID.getVertexID().getDAGId())) { LOG.warn(taskName + " does not belong to " + tezDAGID); continue; String taskAttemptName = entity; TezTaskAttemptID tezAttemptId = TezTaskAttemptID.fromString(taskAttemptName); if (!tezDAGID.equals(tezAttemptId.getTaskID().getVertexID().getDAGId())) { LOG.warn(taskAttemptName + " does not belong to " + tezDAGID); continue;
TezTaskID tezTaskID = TezTaskID.getInstance(tezVertexID, getContext().getTaskIndex()); TezTaskAttemptID tezTaskAttemptID = TezTaskAttemptID.getInstance(tezTaskID, getContext().getTaskAttemptNumber()); jobConf.set(MRInput.TEZ_MAPREDUCE_DAG_ID, tezDAGID.toString()); jobConf.set(MRInput.TEZ_MAPREDUCE_VERTEX_ID, tezVertexID.toString()); jobConf.set(MRInput.TEZ_MAPREDUCE_TASK_ID, tezTaskID.toString()); jobConf.set(MRInput.TEZ_MAPREDUCE_TASK_ATTEMPT_ID, tezTaskAttemptID.toString());
@Override public String toString() { return "vertexName=" + vertexName + ", taskId=" + taskID.toString() + ", scheduledTime=" + scheduledTime + ", launchTime=" + startTime; }
public void fromProto(TaskStartedProto proto) { this.taskID = TezTaskID.fromString(proto.getTaskId()); this.startTime = proto.getLaunchTime(); this.scheduledTime = proto.getScheduledTime(); }
event.getTaskAttemptID().getTaskID().toString()); event.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getApplicationId().toString()); atsEntity.addPrimaryFilter(EntityTypes.TEZ_DAG_ID.name(), event.getTaskAttemptID().getTaskID().getVertexID().getDAGId().toString()); atsEntity.addPrimaryFilter(EntityTypes.TEZ_VERTEX_ID.name(), event.getTaskAttemptID().getTaskID().getVertexID().toString()); atsEntity.addPrimaryFilter(EntityTypes.TEZ_TASK_ID.name(), event.getTaskAttemptID().getTaskID().toString());
boolean containsTask(TezTaskID taskID) { return taskSet.get(taskID.getId()); } }
private static ArrayList<String> convertToStringArrayList( Collection<TezTaskID> collection) { ArrayList<String> list = new ArrayList<String>(collection.size()); for (TezTaskID t : collection) { list.add(t.toString()); } return list; }
public void fromProto(TaskFinishedProto proto) { this.taskID = TezTaskID.fromString(proto.getTaskId()); this.finishTime = proto.getFinishTime(); this.state = TaskState.values()[proto.getState()]; if (proto.hasDiagnostics()) { this.diagnostics = proto.getDiagnostics(); } if (proto.hasSuccessfulTaskAttemptId()) { this.successfulAttemptID = TezTaskAttemptID.fromString(proto.getSuccessfulTaskAttemptId()); } }
private static int vertexNum(TaskInfo taskInfo) { return taskInfo.getAttemptId().getTaskID().getVertexID().getId(); // Sigh... }
private SubmitWorkRequestProto constructSubmitWorkRequest(ContainerId containerId, TaskSpec taskSpec, FragmentRuntimeInfo fragmentRuntimeInfo, String hiveQueryId) throws IOException { SubmitWorkRequestProto.Builder builder = SubmitWorkRequestProto.newBuilder(); builder.setFragmentNumber(taskSpec.getTaskAttemptID().getTaskID().getId()); builder.setAttemptNumber(taskSpec.getTaskAttemptID().getId()); builder.setContainerIdString(containerId.toString()); builder.setAmHost(getAmHostString()); builder.setAmPort(getAddress().getPort()); Preconditions.checkState(currentQueryIdentifierProto.getDagIndex() == taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId()); builder.setCredentialsBinary( getCredentials(getContext().getCurrentDagInfo().getCredentials())); builder.setWorkSpec(VertexOrBinary.newBuilder().setVertex(Converters.constructSignableVertexSpec( taskSpec, currentQueryIdentifierProto, getTokenIdentifier(), user, hiveQueryId)).build()); // Don't call builder.setWorkSpecSignature() - Tez doesn't sign fragments builder.setFragmentRuntimeInfo(fragmentRuntimeInfo); if (scheduler != null) { // May be null in tests // TODO: see javadoc builder.setIsGuaranteed(scheduler.isInitialGuaranteed(taskSpec.getTaskAttemptID())); } return builder.build(); }
private static TimelineEntity convertTaskFinishedEvent(TaskFinishedEvent event) { TimelineEntity atsEntity = new TimelineEntity(); atsEntity.setEntityId(event.getTaskID().toString()); atsEntity.setEntityType(EntityTypes.TEZ_TASK_ID.name()); atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID, event.getTaskID().getVertexID().getDAGId().getApplicationId().toString()); atsEntity.addPrimaryFilter(EntityTypes.TEZ_DAG_ID.name(), event.getTaskID().getVertexID().getDAGId().toString()); atsEntity.addPrimaryFilter(EntityTypes.TEZ_VERTEX_ID.name(), event.getTaskID().getVertexID().toString()); TimelineEvent finishEvt = new TimelineEvent(); finishEvt.setEventType(HistoryEventType.TASK_FINISHED.name()); finishEvt.setTimestamp(event.getFinishTime()); atsEntity.addEvent(finishEvt); atsEntity.addPrimaryFilter(ATSConstants.STATUS, event.getState().name()); atsEntity.addOtherInfo(ATSConstants.FINISH_TIME, event.getFinishTime()); atsEntity.addOtherInfo(ATSConstants.TIME_TAKEN, (event.getFinishTime() - event.getStartTime())); atsEntity.addOtherInfo(ATSConstants.STATUS, event.getState().name()); atsEntity.addOtherInfo(ATSConstants.NUM_FAILED_TASKS_ATTEMPTS, event.getNumFailedAttempts()); if (event.getSuccessfulAttemptID() != null) { atsEntity.addOtherInfo(ATSConstants.SUCCESSFUL_ATTEMPT_ID, event.getSuccessfulAttemptID().toString()); } atsEntity.addOtherInfo(ATSConstants.DIAGNOSTICS, event.getDiagnostics()); atsEntity.addOtherInfo(ATSConstants.COUNTERS, DAGUtils.convertCountersToATSMap(event.getTezCounters())); return atsEntity; }
public static TezTaskAttemptID createTaskAttemptId( QueryIdentifierProto queryIdProto, int vertexIndex, int fragmentNum, int attemptNum) { // Come ride the API roller-coaster! return TezTaskAttemptID.getInstance( TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance( ConverterUtils.toApplicationId( queryIdProto.getApplicationIdString()), queryIdProto.getDagIndex()), vertexIndex), fragmentNum), attemptNum); }