private long getLaunchTime() { long taskLaunchTime = 0; boolean launchTimeSet = false; for (TaskAttempt at : attempts.values()) { // select the least launch time of all attempts long attemptLaunchTime = at.getLaunchTime(); if (attemptLaunchTime != 0 && !launchTimeSet) { // For the first non-zero launch time launchTimeSet = true; taskLaunchTime = attemptLaunchTime; } else if (attemptLaunchTime != 0 && taskLaunchTime > attemptLaunchTime) { taskLaunchTime = attemptLaunchTime; } } if (!launchTimeSet) { return this.scheduledTime; } return taskLaunchTime; }
private long getLaunchTime() { long taskLaunchTime = 0; boolean launchTimeSet = false; for (TaskAttempt at : attempts.values()) { // select the least launch time of all attempts long attemptLaunchTime = at.getLaunchTime(); if (attemptLaunchTime != 0 && !launchTimeSet) { // For the first non-zero launch time launchTimeSet = true; taskLaunchTime = attemptLaunchTime; } else if (attemptLaunchTime != 0 && taskLaunchTime > attemptLaunchTime) { taskLaunchTime = attemptLaunchTime; } } if (!launchTimeSet) { return this.scheduledTime; } return taskLaunchTime; }
private long getLaunchTime() { long taskLaunchTime = 0; boolean launchTimeSet = false; for (TaskAttempt at : attempts.values()) { // select the least launch time of all attempts long attemptLaunchTime = at.getLaunchTime(); if (attemptLaunchTime != 0 && !launchTimeSet) { // For the first non-zero launch time launchTimeSet = true; taskLaunchTime = attemptLaunchTime; } else if (attemptLaunchTime != 0 && taskLaunchTime > attemptLaunchTime) { taskLaunchTime = attemptLaunchTime; } } if (!launchTimeSet) { return this.scheduledTime; } return taskLaunchTime; }
@Override public int compare(TaskAttempt o1, TaskAttempt o2) { if (o1.getFinishTime() == 0 || o2.getFinishTime() == 0) { if (o1.getFinishTime() == 0 && o2.getFinishTime() == 0) { if (o1.getLaunchTime() == 0 || o2.getLaunchTime() == 0) { if (o1.getLaunchTime() == 0 && o2.getLaunchTime() == 0) { return 0; } else { long res = o1.getLaunchTime() - o2.getLaunchTime(); return res > 0 ? -1 : 1; } } else { return (int) (o1.getLaunchTime() - o2.getLaunchTime()); } } else { long res = o1.getFinishTime() - o2.getFinishTime(); return res > 0 ? -1 : 1; } } else { return (int) (o1.getFinishTime() - o2.getFinishTime()); } } });
@Override public int compare(TaskAttempt o1, TaskAttempt o2) { if (o1.getFinishTime() == 0 || o2.getFinishTime() == 0) { if (o1.getFinishTime() == 0 && o2.getFinishTime() == 0) { if (o1.getLaunchTime() == 0 || o2.getLaunchTime() == 0) { if (o1.getLaunchTime() == 0 && o2.getLaunchTime() == 0) { return 0; } else { long res = o1.getLaunchTime() - o2.getLaunchTime(); return res > 0 ? -1 : 1; } } else { return (int) (o1.getLaunchTime() - o2.getLaunchTime()); } } else { long res = o1.getFinishTime() - o2.getFinishTime(); return res > 0 ? -1 : 1; } } else { return (int) (o1.getFinishTime() - o2.getFinishTime()); } } });
@Override public int compare(TaskAttempt o1, TaskAttempt o2) { if (o1.getFinishTime() == 0 || o2.getFinishTime() == 0) { if (o1.getFinishTime() == 0 && o2.getFinishTime() == 0) { if (o1.getLaunchTime() == 0 || o2.getLaunchTime() == 0) { if (o1.getLaunchTime() == 0 && o2.getLaunchTime() == 0) { return 0; } else { long res = o1.getLaunchTime() - o2.getLaunchTime(); return res > 0 ? -1 : 1; } } else { return (int) (o1.getLaunchTime() - o2.getLaunchTime()); } } else { long res = o1.getFinishTime() - o2.getFinishTime(); return res > 0 ? -1 : 1; } } else { return (int) (o1.getFinishTime() - o2.getFinishTime()); } } });
if (taskAttempt.getLaunchTime() != 0 && taskAttempt.getFinishTime() != 0) { attemptRunTime = (int) (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
if (taskAttempt.getLaunchTime() != 0 && taskAttempt.getFinishTime() != 0) { attemptRunTime = (int) (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
private void handleTaskAttemptCompletion(TaskAttemptId attemptId, TaskAttemptCompletionEventStatus status) { TaskAttempt attempt = attempts.get(attemptId); //raise the completion event only if the container is assigned // to nextAttemptNumber if (attempt.getNodeHttpAddress() != null) { TaskAttemptCompletionEvent tce = recordFactory .newRecordInstance(TaskAttemptCompletionEvent.class); tce.setEventId(-1); String scheme = (encryptedShuffle) ? "https://" : "http://"; tce.setMapOutputServerAddress(StringInterner.weakIntern(scheme + attempt.getNodeHttpAddress().split(":")[0] + ":" + attempt.getShufflePort())); tce.setStatus(status); tce.setAttemptId(attempt.getID()); int runTime = 0; if (attempt.getFinishTime() != 0 && attempt.getLaunchTime() !=0) runTime = (int)(attempt.getFinishTime() - attempt.getLaunchTime()); tce.setAttemptRunTime(runTime); //raise the event to job so that it adds the completion event to its //data structures eventHandler.handle(new JobTaskAttemptCompletedEvent(tce)); } }
private void handleTaskAttemptCompletion(TaskAttemptId attemptId, TaskAttemptCompletionEventStatus status) { TaskAttempt attempt = attempts.get(attemptId); //raise the completion event only if the container is assigned // to nextAttemptNumber if (attempt.getNodeHttpAddress() != null) { TaskAttemptCompletionEvent tce = recordFactory .newRecordInstance(TaskAttemptCompletionEvent.class); tce.setEventId(-1); String scheme = (encryptedShuffle) ? "https://" : "http://"; tce.setMapOutputServerAddress(StringInterner.weakIntern(scheme + attempt.getNodeHttpAddress().split(":")[0] + ":" + attempt.getShufflePort())); tce.setStatus(status); tce.setAttemptId(attempt.getID()); int runTime = 0; if (attempt.getFinishTime() != 0 && attempt.getLaunchTime() !=0) runTime = (int)(attempt.getFinishTime() - attempt.getLaunchTime()); tce.setAttemptRunTime(runTime); //raise the event to job so that it adds the completion event to its //data structures eventHandler.handle(new JobTaskAttemptCompletedEvent(tce)); } }
private void handleTaskAttemptCompletion(TaskAttemptId attemptId, TaskAttemptCompletionEventStatus status) { TaskAttempt attempt = attempts.get(attemptId); //raise the completion event only if the container is assigned // to nextAttemptNumber if (attempt.getNodeHttpAddress() != null) { TaskAttemptCompletionEvent tce = recordFactory .newRecordInstance(TaskAttemptCompletionEvent.class); tce.setEventId(-1); String scheme = (encryptedShuffle) ? "https://" : "http://"; tce.setMapOutputServerAddress(StringInterner.weakIntern(scheme + attempt.getNodeHttpAddress().split(":")[0] + ":" + attempt.getShufflePort())); tce.setStatus(status); tce.setAttemptId(attempt.getID()); int runTime = 0; if (attempt.getFinishTime() != 0 && attempt.getLaunchTime() !=0) runTime = (int)(attempt.getFinishTime() - attempt.getLaunchTime()); tce.setAttemptRunTime(runTime); //raise the event to job so that it adds the completion event to its //data structures eventHandler.handle(new JobTaskAttemptCompletedEvent(tce)); } }
private void constructTaskReport() { loadAllTaskAttempts(); this.report = Records.newRecord(TaskReport.class); report.setTaskId(taskId); long minLaunchTime = Long.MAX_VALUE; for(TaskAttempt attempt: attempts.values()) { minLaunchTime = Math.min(minLaunchTime, attempt.getLaunchTime()); } minLaunchTime = minLaunchTime == Long.MAX_VALUE ? -1 : minLaunchTime; report.setStartTime(minLaunchTime); report.setFinishTime(taskInfo.getFinishTime()); report.setTaskState(getState()); report.setProgress(getProgress()); Counters counters = getCounters(); if (counters == null) { counters = EMPTY_COUNTERS; } report.setCounters(TypeConverter.toYarn(counters)); if (successfulAttempt != null) { report.setSuccessfulAttempt(successfulAttempt); } report.addAllDiagnostics(reportDiagnostics); report .addAllRunningAttempts(new ArrayList<TaskAttemptId>(attempts.keySet())); }
private void constructTaskReport() { loadAllTaskAttempts(); this.report = Records.newRecord(TaskReport.class); report.setTaskId(taskId); long minLaunchTime = Long.MAX_VALUE; for(TaskAttempt attempt: attempts.values()) { minLaunchTime = Math.min(minLaunchTime, attempt.getLaunchTime()); } minLaunchTime = minLaunchTime == Long.MAX_VALUE ? -1 : minLaunchTime; report.setStartTime(minLaunchTime); report.setFinishTime(taskInfo.getFinishTime()); report.setTaskState(getState()); report.setProgress(getProgress()); Counters counters = getCounters(); if (counters == null) { counters = EMPTY_COUNTERS; } report.setRawCounters(counters); if (successfulAttempt != null) { report.setSuccessfulAttempt(successfulAttempt); } report.addAllDiagnostics(reportDiagnostics); report .addAllRunningAttempts(new ArrayList<TaskAttemptId>(attempts.keySet())); }
private void constructTaskReport() { loadAllTaskAttempts(); this.report = Records.newRecord(TaskReport.class); report.setTaskId(taskId); long minLaunchTime = Long.MAX_VALUE; for(TaskAttempt attempt: attempts.values()) { minLaunchTime = Math.min(minLaunchTime, attempt.getLaunchTime()); } minLaunchTime = minLaunchTime == Long.MAX_VALUE ? -1 : minLaunchTime; report.setStartTime(minLaunchTime); report.setFinishTime(taskInfo.getFinishTime()); report.setTaskState(getState()); report.setProgress(getProgress()); Counters counters = getCounters(); if (counters == null) { counters = EMPTY_COUNTERS; } report.setCounters(TypeConverter.toYarn(counters)); if (successfulAttempt != null) { report.setSuccessfulAttempt(successfulAttempt); } report.addAllDiagnostics(reportDiagnostics); report .addAllRunningAttempts(new ArrayList<TaskAttemptId>(attempts.keySet())); }
if (attempt.getState() == TaskAttemptState.SUCCEEDED) { numMaps++; avgMapTime += (attempt.getFinishTime() - attempt.getLaunchTime()); numReduces++; avgShuffleTime += (attempt.getShuffleFinishTime() - attempt .getLaunchTime()); avgMergeTime += attempt.getSortFinishTime() - attempt.getShuffleFinishTime();
if (attempt.getState() == TaskAttemptState.SUCCEEDED) { numMaps++; avgMapTime += (attempt.getFinishTime() - attempt.getLaunchTime()); numReduces++; avgShuffleTime += (attempt.getShuffleFinishTime() - attempt .getLaunchTime()); avgMergeTime += attempt.getSortFinishTime() - attempt.getShuffleFinishTime();
if (attempt.getState() == TaskAttemptState.SUCCEEDED) { numMaps++; avgMapTime += (attempt.getFinishTime() - attempt.getLaunchTime()); numReduces++; avgShuffleTime += (attempt.getShuffleFinishTime() - attempt .getLaunchTime()); avgMergeTime += attempt.getSortFinishTime() - attempt.getShuffleFinishTime();
assignedContainerId); assertEquals("startTime wrong", ta.getLaunchTime(), startTime); assertEquals("finishTime wrong", ta.getFinishTime(), finishTime); assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime);
app.waitForState(job, JobState.SUCCEEDED); Assert.assertEquals(mta.getFinishTime(), 11); Assert.assertEquals(mta.getLaunchTime(), 10); Assert.assertEquals(rta.getFinishTime(), 11); Assert.assertEquals(rta.getLaunchTime(), 10); Counters counters = job.getAllCounters();
public void verifyReduceTaskAttemptGeneric(TaskAttempt ta, long shuffleFinishTime, long mergeFinishTime, long elapsedShuffleTime, long elapsedMergeTime, long elapsedReduceTime) { assertEquals("shuffleFinishTime wrong", ta.getShuffleFinishTime(), shuffleFinishTime); assertEquals("mergeFinishTime wrong", ta.getSortFinishTime(), mergeFinishTime); assertEquals("elapsedShuffleTime wrong", ta.getShuffleFinishTime() - ta.getLaunchTime(), elapsedShuffleTime); assertEquals("elapsedMergeTime wrong", ta.getSortFinishTime() - ta.getShuffleFinishTime(), elapsedMergeTime); assertEquals("elapsedReduceTime wrong", ta.getFinishTime() - ta.getSortFinishTime(), elapsedReduceTime); }