@Override public boolean isFinished() { switch (report.getTaskState()) { case SUCCEEDED: case KILLED: case FAILED: return true; } return false; }
@Override public TaskState getState() { return report.getTaskState(); } };
public void waitForState(Task task, TaskState finalState) throws Exception { int timeoutSecs = 0; TaskReport report = task.getReport(); while (!finalState.equals(report.getTaskState()) && timeoutSecs++ < 20) { System.out.println("Task State for " + task.getID() + " is : " + report.getTaskState() + " Waiting for state : " + finalState + " progress : " + report.getProgress()); report = task.getReport(); Thread.sleep(500); } System.out.println("Task State is : " + report.getTaskState()); Assert.assertEquals("Task state is not correct (timedout)", finalState, report.getTaskState()); }
public static TaskReport fromYarn(org.apache.hadoop.mapreduce.v2.api.records.TaskReport report) { String[] diagnostics = null; if (report.getDiagnosticsList() != null) { diagnostics = new String[report.getDiagnosticsCount()]; int i = 0; for (String cs : report.getDiagnosticsList()) { diagnostics[i++] = cs.toString(); } } else { diagnostics = new String[0]; } TaskReport rep = new TaskReport(fromYarn(report.getTaskId()), report.getProgress(), report.getTaskState().toString(), diagnostics, fromYarn(report.getTaskState()), report.getStartTime(), report.getFinishTime(), fromYarn(report.getCounters())); List<org.apache.hadoop.mapreduce.TaskAttemptID> runningAtts = new ArrayList<org.apache.hadoop.mapreduce.TaskAttemptID>(); for (org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId id : report.getRunningAttemptsList()) { runningAtts.add(fromYarn(id)); } rep.setRunningTaskAttemptIds(runningAtts); if (report.getSuccessfulAttempt() != null) { rep.setSuccessfulAttemptId(fromYarn(report.getSuccessfulAttempt())); } return rep; }
public static TaskReport fromYarn(org.apache.hadoop.mapreduce.v2.api.records.TaskReport report) { String[] diagnostics = null; if (report.getDiagnosticsList() != null) { diagnostics = new String[report.getDiagnosticsCount()]; int i = 0; for (String cs : report.getDiagnosticsList()) { diagnostics[i++] = cs.toString(); } } else { diagnostics = new String[0]; } TaskReport rep = new TaskReport(fromYarn(report.getTaskId()), report.getProgress(), report.getTaskState().toString(), diagnostics, fromYarn(report.getTaskState()), report.getStartTime(), report.getFinishTime(), fromYarn(report.getCounters())); List<org.apache.hadoop.mapreduce.TaskAttemptID> runningAtts = new ArrayList<org.apache.hadoop.mapreduce.TaskAttemptID>(); for (org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId id : report.getRunningAttemptsList()) { runningAtts.add(fromYarn(id)); } rep.setRunningTaskAttemptIds(runningAtts); if (report.getSuccessfulAttempt() != null) { rep.setSuccessfulAttemptId(fromYarn(report.getSuccessfulAttempt())); } return rep; }
private void testMRAppHistory(MRApp app) throws Exception { Configuration conf = new Configuration(); Job job = app.submit(conf); app.waitForState(job, JobState.FAILED); Map<TaskId, Task> tasks = job.getTasks(); Assert.assertEquals("Num tasks is not correct", 1, tasks.size()); Task task = tasks.values().iterator().next(); Assert.assertEquals("Task state not correct", TaskState.FAILED, task .getReport().getTaskState()); Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator().next() .getAttempts(); Assert.assertEquals("Num attempts is not correct", 4, attempts.size()); Iterator<TaskAttempt> it = attempts.values().iterator(); TaskAttemptReport report = it.next().getReport(); Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED, report.getTaskAttemptState()); Assert.assertEquals("Diagnostic Information is not Correct", "Test Diagnostic Event", report.getDiagnosticInfo()); report = it.next().getReport(); Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED, report.getTaskAttemptState()); }
public static TaskReport fromYarn(org.apache.hadoop.mapreduce.v2.api.records.TaskReport report) { String[] diagnostics = null; if (report.getDiagnosticsList() != null) { diagnostics = new String[report.getDiagnosticsCount()]; int i = 0; for (String cs : report.getDiagnosticsList()) { diagnostics[i++] = cs.toString(); } } else { diagnostics = new String[0]; } TaskReport rep = new TaskReport(fromYarn(report.getTaskId()), report.getProgress(), report.getTaskState().toString(), diagnostics, fromYarn(report.getTaskState()), report.getStartTime(), report.getFinishTime(), fromYarn(report.getCounters())); List<org.apache.hadoop.mapreduce.TaskAttemptID> runningAtts = new ArrayList<org.apache.hadoop.mapreduce.TaskAttemptID>(); for (org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId id : report.getRunningAttemptsList()) { runningAtts.add(fromYarn(id)); } rep.setRunningTaskAttemptIds(runningAtts); if (report.getSuccessfulAttempt() != null) { rep.setSuccessfulAttemptId(fromYarn(report.getSuccessfulAttempt())); } return rep; }
public static TaskReport fromYarn(org.apache.hadoop.mapreduce.v2.api.records.TaskReport report) { String[] diagnostics = null; if (report.getDiagnosticsList() != null) { diagnostics = new String[report.getDiagnosticsCount()]; int i = 0; for (String cs : report.getDiagnosticsList()) { diagnostics[i++] = cs.toString(); } } else { diagnostics = new String[0]; } TaskReport rep = new TaskReport(fromYarn(report.getTaskId()), report.getProgress(), report.getTaskState().toString(), diagnostics, fromYarn(report.getTaskState()), report.getStartTime(), report.getFinishTime(), fromYarn(report.getCounters())); List<org.apache.hadoop.mapreduce.TaskAttemptID> runningAtts = new ArrayList<org.apache.hadoop.mapreduce.TaskAttemptID>(); for (org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId id : report.getRunningAttemptsList()) { runningAtts.add(fromYarn(id)); } rep.setRunningTaskAttemptIds(runningAtts); if (report.getSuccessfulAttempt() != null) { rep.setSuccessfulAttemptId(fromYarn(report.getSuccessfulAttempt())); } return rep; }
@Test //All Task attempts are timed out, leading to Job failure public void testTimedOutTask() throws Exception { MRApp app = new TimeOutTaskMRApp(1, 0); Configuration conf = new Configuration(); int maxAttempts = 2; conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, maxAttempts); // disable uberization (requires entire job to be reattempted, so max for // subtask attempts is overridden to 1) conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); Job job = app.submit(conf); app.waitForState(job, JobState.FAILED); Map<TaskId,Task> tasks = job.getTasks(); Assert.assertEquals("Num tasks is not correct", 1, tasks.size()); Task task = tasks.values().iterator().next(); Assert.assertEquals("Task state not correct", TaskState.FAILED, task.getReport().getTaskState()); Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator().next().getAttempts(); Assert.assertEquals("Num attempts is not correct", maxAttempts, attempts.size()); for (TaskAttempt attempt : attempts.values()) { Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED, attempt.getReport().getTaskAttemptState()); } }
@Test //First attempt is failed and second attempt is passed //The job succeeds. public void testFailTask() throws Exception { MRApp app = new MockFirstFailingAttemptMRApp(1, 0); Configuration conf = new Configuration(); // this test requires two task attempts, but uberization overrides max to 1 conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); Job job = app.submit(conf); app.waitForState(job, JobState.SUCCEEDED); Map<TaskId,Task> tasks = job.getTasks(); Assert.assertEquals("Num tasks is not correct", 1, tasks.size()); Task task = tasks.values().iterator().next(); Assert.assertEquals("Task state not correct", TaskState.SUCCEEDED, task.getReport().getTaskState()); Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator().next().getAttempts(); Assert.assertEquals("Num attempts is not correct", 2, attempts.size()); //one attempt must be failed //and another must have succeeded Iterator<TaskAttempt> it = attempts.values().iterator(); Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED, it.next().getReport().getTaskAttemptState()); Assert.assertEquals("Attempt state not correct", TaskAttemptState.SUCCEEDED, it.next().getReport().getTaskAttemptState()); }
TaskReport report = mock(TaskReport.class); when(report.getProgress()).thenReturn(0.7f); when(report.getTaskState()).thenReturn(TaskState.SUCCEEDED); when(report.getStartTime()).thenReturn(100001L); when(report.getFinishTime()).thenReturn(100011L);
task1.getReport().getTaskState()); Assert.assertEquals("Task state not correct", TaskState.SUCCEEDED, task2.getReport().getTaskState()); Map<TaskAttemptId, TaskAttempt> attempts = task1.getAttempts(); Assert.assertEquals("No of attempts is not correct", 1,
Task task = tasks.values().iterator().next(); Assert.assertEquals("Task state not correct", TaskState.KILLED, task.getReport().getTaskState()); Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator().next().getAttempts();
task1.getReport().getTaskState()); Assert.assertEquals("Task state not correct", TaskState.SUCCEEDED, task2.getReport().getTaskState());
TaskState.NEW, reduceTask.getReport().getTaskState());
TaskState.RUNNING, reduceTask.getReport().getTaskState());
public TaskInfo(Task task) { TaskType ttype = task.getType(); this.type = ttype.toString(); TaskReport report = task.getReport(); this.startTime = report.getStartTime(); this.finishTime = report.getFinishTime(); this.state = report.getTaskState(); this.elapsedTime = Times.elapsed(this.startTime, this.finishTime, this.state == TaskState.RUNNING); if (this.elapsedTime == -1) { this.elapsedTime = 0; } this.progress = report.getProgress() * 100; this.status = report.getStatus(); this.id = MRApps.toString(task.getID()); this.taskNum = task.getID().getId(); this.successful = getSuccessfulAttempt(task); if (successful != null) { this.successfulAttempt = MRApps.toString(successful.getID()); } else { this.successfulAttempt = ""; } }
public TaskInfo(Task task) { TaskType ttype = task.getType(); this.type = ttype.toString(); TaskReport report = task.getReport(); this.startTime = report.getStartTime(); this.finishTime = report.getFinishTime(); this.state = report.getTaskState(); this.elapsedTime = Times.elapsed(this.startTime, this.finishTime, this.state == TaskState.RUNNING); if (this.elapsedTime == -1) { this.elapsedTime = 0; } this.progress = report.getProgress() * 100; this.status = report.getStatus(); this.id = MRApps.toString(task.getID()); this.taskNum = task.getID().getId(); this.successful = getSuccessfulAttempt(task); if (successful != null) { this.successfulAttempt = MRApps.toString(successful.getID()); } else { this.successfulAttempt = ""; } }
public TaskInfo(Task task) { TaskType ttype = task.getType(); this.type = ttype.toString(); TaskReport report = task.getReport(); this.startTime = report.getStartTime(); this.finishTime = report.getFinishTime(); this.state = report.getTaskState(); this.elapsedTime = Times.elapsed(this.startTime, this.finishTime, this.state == TaskState.RUNNING); if (this.elapsedTime == -1) { this.elapsedTime = 0; } this.progress = report.getProgress() * 100; this.status = report.getStatus(); this.id = MRApps.toString(task.getID()); this.taskNum = task.getID().getId(); this.successful = getSuccessfulAttempt(task); if (successful != null) { this.successfulAttempt = MRApps.toString(successful.getID()); } else { this.successfulAttempt = ""; } }
public void verifyTaskGeneric(Task task, String id, String state, String type, String successfulAttempt, long startTime, long finishTime, long elapsedTime, float progress, String status) { TaskId taskid = task.getID(); String tid = MRApps.toString(taskid); TaskReport report = task.getReport(); WebServicesTestUtils.checkStringMatch("id", tid, id); WebServicesTestUtils.checkStringMatch("type", task.getType().toString(), type); WebServicesTestUtils.checkStringMatch("state", report.getTaskState() .toString(), state); // not easily checked without duplicating logic, just make sure its here assertNotNull("successfulAttempt null", successfulAttempt); assertEquals("startTime wrong", report.getStartTime(), startTime); assertEquals("finishTime wrong", report.getFinishTime(), finishTime); assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime); assertEquals("progress wrong", report.getProgress() * 100, progress, 1e-3f); assertEquals("status wrong", report.getStatus(), status); }