/** * Get the user defined {@link RawComparator} comparator for * grouping keys of inputs to the combiner. * * @return comparator set by the user for grouping values. * @see Job#setCombinerKeyGroupingComparatorClass(Class) for details. */ public RawComparator<?> getCombinerKeyGroupingComparator() { return getSortComparator(); }
public TaskAttemptContextImpl(Configuration conf, TezCounters tezCounters, long clusterId, int vertexIndex, int appId, int taskIndex, int taskAttemptNumber, boolean isMap, Reporter reporter) { // TODO NEWTEZ Can the jt Identifier string be taskContext.getUniqueId ? this(conf, createMockTaskAttemptID(clusterId, vertexIndex, appId, taskIndex, taskAttemptNumber, isMap), tezCounters, reporter); }
/** * Set the current status of the task to the given string. */ @Override public void setStatus(String status) { setStatusString(status); // Nothing to do until InputContext supports some kind of custom string // diagnostics. } }
/** * Get the timestamps of the archives. Used by internal * DistributedCache and MapReduce code. * @return a string array of timestamps * @throws IOException */ public String[] getArchiveTimestamps() { return toTimestampStrs(DistributedCache.getArchiveTimestamps(conf)); }
private TaskAttemptContext createTaskAttemptContext(TaskAttemptID attemptId) { return new TaskAttemptContextImpl(this.jobConf, attemptId, getContext().getCounters(), isMapperOutput, null); }
public static TaskID toMRTaskIdForOutput(TezTaskID taskid) { return org.apache.tez.mapreduce.hadoop.mapreduce.TaskAttemptContextImpl .createMockTaskAttemptIDFromTezTaskId(taskid, (taskid.getVertexID().getId() == 0)); }
public void initTask(LogicalOutput output) throws IOException, InterruptedException { // By this time output has been initialized this.output = output; if (output instanceof MROutputLegacy) { committer = ((MROutputLegacy)output).getOutputCommitter(); } this.mrReporter = new MRTaskReporter(processorContext); this.useNewApi = jobConf.getUseNewMapper(); TezDAGID dagId = IDConverter.fromMRTaskAttemptId(taskAttemptId).getTaskID() .getVertexID().getDAGId(); this.jobContext = new JobContextImpl(jobConf, dagId, mrReporter); this.taskAttemptContext = new TaskAttemptContextImpl(jobConf, taskAttemptId, mrReporter); localizeConfiguration(jobConf); }
new MapContextImpl( job, taskAttemptId, input, output,
public static void setMRProperties( ProcessorContext context, Configuration config, boolean isMapperOutput ) { TaskAttemptID taskAttemptId = org.apache.tez.mapreduce.hadoop.mapreduce.TaskAttemptContextImpl .createMockTaskAttemptID( context.getApplicationId().getClusterTimestamp(), context.getTaskVertexIndex(), context.getApplicationId().getId(), context.getTaskIndex(), context.getTaskAttemptNumber(), isMapperOutput ); config.set( JobContext.TASK_ATTEMPT_ID, taskAttemptId.toString() ); config.set( JobContext.TASK_ID, taskAttemptId.getTaskID().toString() ); config.setBoolean( JobContext.TASK_ISMAP, isMapperOutput ); config.setInt( JobContext.TASK_PARTITION, taskAttemptId.getTaskID().getId() ); } }
/** * Get the timestamps of the files. Used by internal * DistributedCache and MapReduce code. * @return a string array of timestamps * @throws IOException */ public String[] getFileTimestamps() { return toTimestampStrs(DistributedCache.getFileTimestamps(conf)); }
public MRReaderMapReduce(JobConf jobConf, InputSplit inputSplit, TezCounters tezCounters, TezCounter inputRecordCounter, long clusterId, int vertexIndex, int appId, int taskIndex, int taskAttemptNumber, InputContext context) throws IOException { super(context); this.inputRecordCounter = inputRecordCounter; this.taskAttemptContext = new TaskAttemptContextImpl(jobConf, tezCounters, clusterId, vertexIndex, appId, taskIndex, taskAttemptNumber, true, null); Class<? extends org.apache.hadoop.mapreduce.InputFormat<?, ?>> inputFormatClazz; try { inputFormatClazz = taskAttemptContext.getInputFormatClass(); } catch (ClassNotFoundException e) { throw new IOException("Unable to instantiate InputFormat class", e); } inputFormat = ReflectionUtils.newInstance(inputFormatClazz, jobConf); if (inputSplit != null) { this.inputSplit = inputSplit; setupNewRecordReader(); } }
public static void setMRProperties( ProcessorContext context, Configuration config, boolean isMapperOutput ) { TaskAttemptID taskAttemptId = org.apache.tez.mapreduce.hadoop.mapreduce.TaskAttemptContextImpl .createMockTaskAttemptID( context.getApplicationId().getClusterTimestamp(), context.getTaskVertexIndex(), context.getApplicationId().getId(), context.getTaskIndex(), context.getTaskAttemptNumber(), isMapperOutput ); config.set( JobContext.TASK_ATTEMPT_ID, taskAttemptId.toString() ); config.set( JobContext.TASK_ID, taskAttemptId.getTaskID().toString() ); config.setBoolean( JobContext.TASK_ISMAP, isMapperOutput ); config.setInt( JobContext.TASK_PARTITION, taskAttemptId.getTaskID().getId() ); } }
getContext().getDAGAttemptNumber()); TaskAttemptID taskAttemptId = org.apache.tez.mapreduce.hadoop.mapreduce.TaskAttemptContextImpl .createMockTaskAttemptID(getContext().getApplicationId().getClusterTimestamp(), getContext().getTaskVertexIndex(), getContext().getApplicationId().getId(), getContext().getTaskIndex(), getContext().getTaskAttemptNumber(), isMapperOutput);
.createMockTaskAttemptID(getContext().getApplicationId().getClusterTimestamp(), getContext().getTaskVertexIndex(), getContext().getApplicationId().getId(), getContext().getTaskIndex(), getContext().getTaskAttemptNumber(), isMapperOutput);