public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
public static TezTaskAttemptID createTaskAttemptId( QueryIdentifierProto queryIdProto, int vertexIndex, int fragmentNum, int attemptNum) { // Come ride the API roller-coaster! return TezTaskAttemptID.getInstance( TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance( ConverterUtils.toApplicationId( queryIdProto.getApplicationIdString()), queryIdProto.getDagIndex()), vertexIndex), fragmentNum), attemptNum); }
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
public static TezTaskAttemptID createTaskAttemptId(TaskContext ctx) { // Come ride the API roller-coaster #2! The best part is that ctx has TezTaskAttemptID inside. return TezTaskAttemptID.getInstance( TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance( ctx.getApplicationId(), ctx.getDagIdentifier()), ctx.getTaskVertexIndex()), ctx.getTaskIndex()), ctx.getTaskAttemptNumber()); }
TezDAGID tezDagId = TezDAGID.getInstance(appId, 300); TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400); TezTaskID tezTaskId = TezTaskID.getInstance(tezVertexId, 500); TezTaskAttemptID tezTaskAttemptId = TezTaskAttemptID.getInstance(tezTaskId, 600);
TezDAGID tezDagId = TezDAGID.getInstance(appId, 300); TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400); TezTaskID tezTaskId = TezTaskID.getInstance(tezVertexId, 500); TezTaskAttemptID tezTaskAttemptId = TezTaskAttemptID.getInstance(tezTaskId, 600);
public static TezTaskID newTaskId(TezDAGID dagId, int vertexId, int taskId) { return TezTaskID.getInstance(newVertexID(dagId, vertexId), taskId); }
@Override public Task getTask(int taskIndex) { return getTask(TezTaskID.getInstance(this.vertexId, taskIndex)); }
public static TezTaskID fromString(String taskIdStr) { try { int pos1 = taskIdStr.indexOf(SEPARATOR); int pos2 = taskIdStr.indexOf(SEPARATOR, pos1 + 1); int pos3 = taskIdStr.indexOf(SEPARATOR, pos2 + 1); int pos4 = taskIdStr.indexOf(SEPARATOR, pos3 + 1); int pos5 = taskIdStr.indexOf(SEPARATOR, pos4 + 1); String rmId = taskIdStr.substring(pos1 + 1, pos2); int appId = Integer.parseInt(taskIdStr.substring(pos2 + 1, pos3)); int dagId = Integer.parseInt(taskIdStr.substring(pos3 + 1, pos4)); int vId = Integer.parseInt(taskIdStr.substring(pos4 + 1, pos5)); int id = Integer.parseInt(taskIdStr.substring(pos5 + 1)); return TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance(rmId, appId, dagId), vId), id); } catch (Exception e) { e.printStackTrace(); } return null; }
public static TezTaskID readTezTaskID(DataInput in) throws IOException { TezVertexID vertexID = TezVertexID.readTezVertexID(in); int taskIdInt = TezID.readID(in); return getInstance(vertexID, taskIdInt); }
public static TezTaskAttemptID fromString(String taIdStr) { try { int pos1 = taIdStr.indexOf(SEPARATOR); int pos2 = taIdStr.indexOf(SEPARATOR, pos1 + 1); int pos3 = taIdStr.indexOf(SEPARATOR, pos2 + 1); int pos4 = taIdStr.indexOf(SEPARATOR, pos3 + 1); int pos5 = taIdStr.indexOf(SEPARATOR, pos4 + 1); int pos6 = taIdStr.indexOf(SEPARATOR, pos5 + 1); String rmId = taIdStr.substring(pos1 + 1, pos2); int appId = Integer.parseInt(taIdStr.substring(pos2 + 1, pos3)); int dagId = Integer.parseInt(taIdStr.substring(pos3 + 1, pos4)); int vId = Integer.parseInt(taIdStr.substring(pos4 + 1, pos5)); int taskId = Integer.parseInt(taIdStr.substring(pos5 + 1, pos6)); int id = Integer.parseInt(taIdStr.substring(pos6 + 1)); return TezTaskAttemptID.getInstance( TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance(rmId, appId, dagId), vId), taskId), id); } catch (Exception e) { e.printStackTrace(); } return null; }
public static TezTaskID fromMRTaskId(org.apache.hadoop.mapreduce.TaskID taskid) { return TezTaskID.getInstance( TezVertexID.getInstance(fromMRJobId(taskid.getJobID()), (taskid.getTaskType() == TaskType.MAP ? 0 : 1) ), taskid.getId()); }
this.attempts = Collections.emptyMap(); maxFailedAttempts = vertex.getVertexConfig().getMaxFailedTaskAttempts(); taskId = TezTaskID.getInstance(vertexId, taskIndex); this.taskCommunicatorManagerInterface = taskCommunicatorManagerInterface; this.taskHeartbeatHandler = thh;
TezTaskID taskId = TezTaskID.getInstance(vertexId, task.getTaskIndex()); TaskSpec baseTaskSpec = createRemoteTaskSpec(taskId.getId()); boolean fromRecovery = recoveryData == null ? false : recoveryData.getTaskRecoveryData(taskId) != null;
TezTaskID tezTaskID = TezTaskID.getInstance(tezVertexID, getContext().getTaskIndex()); TezTaskAttemptID tezTaskAttemptID = TezTaskAttemptID.getInstance(tezTaskID, getContext().getTaskAttemptNumber()); jobConf.set(MRInput.TEZ_MAPREDUCE_DAG_ID, tezDAGID.toString());