outputFormat.checkOutputSpecs(dfs, jobConf); JobID jobID = new JobID(new SimpleDateFormat("yyyyMMddHHmm").format(new Date()), 0); TaskAttemptID taId = new TaskAttemptID(new TaskID(jobID, true, 0), 0); jobConf.set("mapred.tip.id", taId.getTaskID().toString()); jobConf.set("mapred.task.id", taId.toString());
/** * Creates Hadoop attempt ID. * * @return Attempt ID. */ public TaskAttemptID attemptId() { TaskID tid = new TaskID(jobCtx.getJobID(), taskType(taskInfo().type()), taskInfo().taskNumber()); return new TaskAttemptID(tid, taskInfo().attempt()); }
private MapTaskCompletionEventsUpdate getMockedCompletionEventsUpdate( int startIdx, int numEvents) { ArrayList<TaskCompletionEvent> tceList = new ArrayList<TaskCompletionEvent>(numEvents); for (int i = 0; i < numEvents; ++i) { int eventIdx = startIdx + i; TaskCompletionEvent tce = new TaskCompletionEvent(eventIdx, new TaskAttemptID("12345", 1, TaskType.MAP, eventIdx, 0), eventIdx, true, TaskCompletionEvent.Status.SUCCEEDED, "http://somehost:8888"); tceList.add(tce); } TaskCompletionEvent[] events = {}; return new MapTaskCompletionEventsUpdate(tceList.toArray(events), false); }
throws IOException, InterruptedException { final int MAX_EVENTS_TO_FETCH = 100; TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.REDUCE, 1, 1);
MapOutputFile mof = new MROutputFiles(); mof.setConf(conf); TaskAttemptID attemptId = new TaskAttemptID("12345", 1, TaskType.MAP, 1, 1); MapTask mockTask = mock(MapTask.class); doReturn(mof).when(mockTask).getMapOutputFile();
TaskAttemptID succeedAttempt1ID = new TaskAttemptID( new org.apache.hadoop.mapred.TaskID( new JobID("test", 0), TaskType.MAP, 0), 0); TaskAttemptID succeedAttempt2ID = new TaskAttemptID( new org.apache.hadoop.mapred.TaskID( new JobID("test", 0), TaskType.MAP, 0), 1); TaskAttemptID succeedAttempt3ID = new TaskAttemptID( new org.apache.hadoop.mapred.TaskID( new JobID("test", 0), TaskType.MAP, 1), 0);
TaskAttemptID failedAttemptID = new TaskAttemptID( new org.apache.hadoop.mapred.TaskID( new JobID("test",0), TaskType.MAP, 0), 0); TaskAttemptID succeedAttemptID = new TaskAttemptID( new org.apache.hadoop.mapred.TaskID( new JobID("test",0), TaskType.MAP, 1), 1);
/** * test TaskAttemptID */ @SuppressWarnings( "deprecation" ) @Test (timeout=5000) public void testTaskAttemptID (){ TaskAttemptID task = new TaskAttemptID("001",2,true,3,4); assertEquals("attempt_001_0002_m_000003_4", TaskAttemptID.getTaskAttemptIDsPattern("001", 2, true, 3, 4)); assertEquals("task_001_0002_m_000003", task.getTaskID().toString()); assertEquals("attempt_001_0001_r_000002_3",TaskAttemptID.getTaskAttemptIDsPattern("001", 1, TaskType.REDUCE, 2, 3)); assertEquals("001_0001_m_000001_2", TaskAttemptID.getTaskAttemptIDsPatternWOPrefix("001",1, TaskType.MAP, 1, 2).toString()); }
@Deprecated public static TaskAttemptID read(DataInput in) throws IOException { TaskAttemptID taskId = new TaskAttemptID(); taskId.readFields(in); return taskId; }
private static void addTaskAttemptInfo( JobHistoryParser.TaskInfo task, int id) { JobHistoryParser.TaskAttemptInfo attempt = new JobHistoryParser.TaskAttemptInfo(); attempt.attemptId = new TaskAttemptID( TaskID.downgrade(task.getTaskId()), id); attempt.startTime = task.getStartTime(); attempt.finishTime = task.getFinishTime(); attempt.shuffleFinishTime = task.getFinishTime(); attempt.sortFinishTime = task.getFinishTime(); attempt.mapFinishTime = task.getFinishTime(); attempt.status = task.getTaskStatus(); attempt.taskType = task.getTaskType(); attempt.trackerName = "localhost"; attempt.httpPort = 1234; attempt.hostname = "localhost"; task.attemptsMap.put(attempt.getAttemptId(), attempt); } }
/** * Constructor * * @param conf Configuration */ public PerThread(Configuration conf) { this.conf = HiveUtils.newHiveConf(conf, OutputCmd.class); this.taskID = new TaskAttemptID("hiveio_output", 42, true, (int) Thread.currentThread().getId(), 0); }
@Deprecated public static TaskAttemptID read(DataInput in) throws IOException { TaskAttemptID taskId = new TaskAttemptID(); taskId.readFields(in); return taskId; }
null, progress, context.getShuffledMapsCounter(), context.getReduceShuffleBytes(), context.getFailedShuffleCounter()); TaskAttemptID attemptID0 = new TaskAttemptID( new org.apache.hadoop.mapred.TaskID( new JobID("test",0), TaskType.MAP, 0), 0); TaskAttemptID attemptID1 = new TaskAttemptID( new org.apache.hadoop.mapred.TaskID( new JobID("test",0), TaskType.MAP, 1), 1); TaskAttemptID attemptID2 = new TaskAttemptID( new org.apache.hadoop.mapred.TaskID( new JobID("test",0), TaskType.MAP, 2), 2); TaskAttemptID attemptID3 = new TaskAttemptID( new org.apache.hadoop.mapred.TaskID( new JobID("test",0), TaskType.MAP, 3), 3); TaskAttemptID attemptID4 = new TaskAttemptID( new org.apache.hadoop.mapred.TaskID( new JobID("test",0), TaskType.MAP, 4), 4); TaskAttemptID attemptID5 = new TaskAttemptID( new org.apache.hadoop.mapred.TaskID( new JobID("test",0), TaskType.MAP, 5), 5); TaskAttemptID attemptID6 = new TaskAttemptID( new org.apache.hadoop.mapred.TaskID( new JobID("test",0), TaskType.MAP, 6), 6);
Progress progress = new Progress(); TaskAttemptID reduceId = new TaskAttemptID("314159", 0, TaskType.REDUCE, 0, 0); ShuffleSchedulerImpl scheduler = new ShuffleSchedulerImpl(job, status,
private TaskAttemptID getTaskAttemptID(TaskType type) { JobID jobId = getJobID(); return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(), type, ++taskCounter, 0); } }
public static org.apache.hadoop.mapred.TaskAttemptID fromYarn( TaskAttemptId id) { return new org.apache.hadoop.mapred.TaskAttemptID(fromYarn(id.getTaskId()), id.getId()); }
/** * test deprecated methods of TaskCompletionEvent */ @SuppressWarnings("deprecation") @Test (timeout=5000) public void testTaskCompletionEvent() { TaskAttemptID taid = new TaskAttemptID("001", 1, TaskType.REDUCE, 2, 3); TaskCompletionEvent template = new TaskCompletionEvent(12, taid, 13, true, Status.SUCCEEDED, "httptracker"); TaskCompletionEvent testEl = TaskCompletionEvent.downgrade(template); testEl.setTaskAttemptId(taid); testEl.setTaskTrackerHttp("httpTracker"); testEl.setTaskId("attempt_001_0001_m_000002_04"); assertEquals("attempt_001_0001_m_000002_4",testEl.getTaskId()); testEl.setTaskStatus(Status.OBSOLETE); assertEquals(Status.OBSOLETE.toString(), testEl.getStatus().toString()); testEl.setTaskRunTime(20); assertEquals(testEl.getTaskRunTime(), 20); testEl.setEventId(16); assertEquals(testEl.getEventId(), 16); }
public Task() { taskStatus = TaskStatus.createTaskStatus(isMapTask()); taskId = new TaskAttemptID(); spilledRecordsCounter = counters.findCounter(TaskCounter.SPILLED_RECORDS); failedShuffleCounter = counters.findCounter(TaskCounter.FAILED_SHUFFLE); mergedMapOutputsCounter = counters.findCounter(TaskCounter.MERGED_MAP_OUTPUTS); gcUpdater = new GcTimeUpdater(); }
public Task() { taskStatus = TaskStatus.createTaskStatus(isMapTask()); taskId = new TaskAttemptID(); spilledRecordsCounter = counters.findCounter(TaskCounter.SPILLED_RECORDS); failedShuffleCounter = counters.findCounter(TaskCounter.FAILED_SHUFFLE); mergedMapOutputsCounter = counters.findCounter(TaskCounter.MERGED_MAP_OUTPUTS); gcUpdater = new GcTimeUpdater(); }
public Task() { taskStatus = TaskStatus.createTaskStatus(isMapTask()); taskId = new TaskAttemptID(); spilledRecordsCounter = counters.findCounter(TaskCounter.SPILLED_RECORDS); failedShuffleCounter = counters.findCounter(TaskCounter.FAILED_SHUFFLE); mergedMapOutputsCounter = counters.findCounter(TaskCounter.MERGED_MAP_OUTPUTS); gcUpdater = new GcTimeUpdater(); }