public void updateAndLogIfChanged(String msgPrefix) { boolean changed = false; // synchronized to fix findbug warnings synchronized (RMContainerAllocator.this) { changed |= (numPendingReduces != pendingReduces.size()); numPendingReduces = pendingReduces.size(); changed |= (numScheduledMaps != scheduledRequests.maps.size()); numScheduledMaps = scheduledRequests.maps.size(); changed |= (numScheduledReduces != scheduledRequests.reduces.size()); numScheduledReduces = scheduledRequests.reduces.size(); changed |= (numAssignedMaps != assignedRequests.maps.size()); numAssignedMaps = assignedRequests.maps.size(); changed |= (numAssignedReduces != assignedRequests.reduces.size()); numAssignedReduces = assignedRequests.reduces.size(); changed |= (numCompletedMaps != getJob().getCompletedMaps()); numCompletedMaps = getJob().getCompletedMaps(); changed |= (numCompletedReduces != getJob().getCompletedReduces()); numCompletedReduces = getJob().getCompletedReduces(); changed |= (numContainersAllocated != containersAllocated); numContainersAllocated = containersAllocated; changed |= (numContainersReleased != containersReleased); numContainersReleased = containersReleased; } if (changed) { log(msgPrefix); } }
public void updateAndLogIfChanged(String msgPrefix) { boolean changed = false; // synchronized to fix findbug warnings synchronized (RMContainerAllocator.this) { changed |= (numPendingReduces != pendingReduces.size()); numPendingReduces = pendingReduces.size(); changed |= (numScheduledMaps != scheduledRequests.maps.size()); numScheduledMaps = scheduledRequests.maps.size(); changed |= (numScheduledReduces != scheduledRequests.reduces.size()); numScheduledReduces = scheduledRequests.reduces.size(); changed |= (numAssignedMaps != assignedRequests.maps.size()); numAssignedMaps = assignedRequests.maps.size(); changed |= (numAssignedReduces != assignedRequests.reduces.size()); numAssignedReduces = assignedRequests.reduces.size(); changed |= (numCompletedMaps != getJob().getCompletedMaps()); numCompletedMaps = getJob().getCompletedMaps(); changed |= (numCompletedReduces != getJob().getCompletedReduces()); numCompletedReduces = getJob().getCompletedReduces(); changed |= (numContainersAllocated != containersAllocated); numContainersAllocated = containersAllocated; changed |= (numContainersReleased != containersReleased); numContainersReleased = containersReleased; } if (changed) { log(msgPrefix); } }
public void updateAndLogIfChanged(String msgPrefix) { boolean changed = false; // synchronized to fix findbug warnings synchronized (RMContainerAllocator.this) { changed |= (numPendingReduces != pendingReduces.size()); numPendingReduces = pendingReduces.size(); changed |= (numScheduledMaps != scheduledRequests.maps.size()); numScheduledMaps = scheduledRequests.maps.size(); changed |= (numScheduledReduces != scheduledRequests.reduces.size()); numScheduledReduces = scheduledRequests.reduces.size(); changed |= (numAssignedMaps != assignedRequests.maps.size()); numAssignedMaps = assignedRequests.maps.size(); changed |= (numAssignedReduces != assignedRequests.reduces.size()); numAssignedReduces = assignedRequests.reduces.size(); changed |= (numCompletedMaps != getJob().getCompletedMaps()); numCompletedMaps = getJob().getCompletedMaps(); changed |= (numCompletedReduces != getJob().getCompletedReduces()); numCompletedReduces = getJob().getCompletedReduces(); changed |= (numContainersAllocated != containersAllocated); numContainersAllocated = containersAllocated; changed |= (numContainersReleased != containersReleased); numContainersReleased = containersReleased; } if (changed) { log(msgPrefix); } }
private void printStat(Job job, long startTime) throws Exception { long currentTime = System.currentTimeMillis(); Runtime.getRuntime().gc(); long mem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); System.out.println("JobState:" + job.getState() + " CompletedMaps:" + job.getCompletedMaps() + " CompletedReduces:" + job.getCompletedReduces() + " Memory(total-free)(KB):" + mem/1024 + " ElapsedTime(ms):" + (currentTime - startTime)); }
@Override protected synchronized void heartbeat() throws Exception { scheduleStats.updateAndLogIfChanged("Before Scheduling: "); List<Container> allocatedContainers = getResources(); if (allocatedContainers != null && allocatedContainers.size() > 0) { scheduledRequests.assign(allocatedContainers); } int completedMaps = getJob().getCompletedMaps(); int completedTasks = completedMaps + getJob().getCompletedReduces(); if ((lastCompletedTasks != completedTasks) || (scheduledRequests.maps.size() > 0)) { lastCompletedTasks = completedTasks; recalculateReduceSchedule = true; } if (recalculateReduceSchedule) { boolean reducerPreempted = preemptReducesIfNeeded(); if (!reducerPreempted) { // Only schedule new reducers if no reducer preemption happens for // this heartbeat scheduleReduces(getJob().getTotalMaps(), completedMaps, scheduledRequests.maps.size(), scheduledRequests.reduces.size(), assignedRequests.maps.size(), assignedRequests.reduces.size(), mapResourceRequest, reduceResourceRequest, pendingReduces.size(), maxReduceRampupLimit, reduceSlowStart); } recalculateReduceSchedule = false; } scheduleStats.updateAndLogIfChanged("After Scheduling: "); }
@Override protected synchronized void heartbeat() throws Exception { scheduleStats.updateAndLogIfChanged("Before Scheduling: "); List<Container> allocatedContainers = getResources(); if (allocatedContainers != null && allocatedContainers.size() > 0) { scheduledRequests.assign(allocatedContainers); } int completedMaps = getJob().getCompletedMaps(); int completedTasks = completedMaps + getJob().getCompletedReduces(); if ((lastCompletedTasks != completedTasks) || (scheduledRequests.maps.size() > 0)) { lastCompletedTasks = completedTasks; recalculateReduceSchedule = true; } if (recalculateReduceSchedule) { boolean reducerPreempted = preemptReducesIfNeeded(); if (!reducerPreempted) { // Only schedule new reducers if no reducer preemption happens for // this heartbeat scheduleReduces(getJob().getTotalMaps(), completedMaps, scheduledRequests.maps.size(), scheduledRequests.reduces.size(), assignedRequests.maps.size(), assignedRequests.reduces.size(), mapResourceRequest, reduceResourceRequest, pendingReduces.size(), maxReduceRampupLimit, reduceSlowStart); } recalculateReduceSchedule = false; } scheduleStats.updateAndLogIfChanged("After Scheduling: "); }
@Override protected synchronized void heartbeat() throws Exception { scheduleStats.updateAndLogIfChanged("Before Scheduling: "); List<Container> allocatedContainers = getResources(); if (allocatedContainers != null && allocatedContainers.size() > 0) { scheduledRequests.assign(allocatedContainers); } int completedMaps = getJob().getCompletedMaps(); int completedTasks = completedMaps + getJob().getCompletedReduces(); if ((lastCompletedTasks != completedTasks) || (scheduledRequests.maps.size() > 0)) { lastCompletedTasks = completedTasks; recalculateReduceSchedule = true; } if (recalculateReduceSchedule) { boolean reducerPreempted = preemptReducesIfNeeded(); if (!reducerPreempted) { // Only schedule new reducers if no reducer preemption happens for // this heartbeat scheduleReduces(getJob().getTotalMaps(), completedMaps, scheduledRequests.maps.size(), scheduledRequests.reduces.size(), assignedRequests.maps.size(), assignedRequests.reduces.size(), mapResourceRequest, reduceResourceRequest, pendingReduces.size(), maxReduceRampupLimit, reduceSlowStart); } recalculateReduceSchedule = false; } scheduleStats.updateAndLogIfChanged("After Scheduling: "); }
JobUnsuccessfulCompletionEvent jucEvent = new JobUnsuccessfulCompletionEvent(TypeConverter.fromYarn(toClose), System.currentTimeMillis(), job.getCompletedMaps(), job.getCompletedReduces(), createJobStateForJobUnsuccessfulCompletionEvent(
@Override public long thresholdRuntime(TaskId taskID) { JobId jobID = taskID.getJobId(); Job job = context.getJob(jobID); TaskType type = taskID.getTaskType(); DataStatistics statistics = dataStatisticsForTask(taskID); int completedTasksOfType = type == TaskType.MAP ? job.getCompletedMaps() : job.getCompletedReduces(); int totalTasksOfType = type == TaskType.MAP ? job.getTotalMaps() : job.getTotalReduces(); if (completedTasksOfType < MINIMUM_COMPLETE_NUMBER_TO_SPECULATE || (((float)completedTasksOfType) / totalTasksOfType) < MINIMUM_COMPLETE_PROPORTION_TO_SPECULATE ) { return Long.MAX_VALUE; } long result = statistics == null ? Long.MAX_VALUE : (long)statistics.outlier(slowTaskRelativeTresholds.get(job)); return result; }
@Override public long thresholdRuntime(TaskId taskID) { JobId jobID = taskID.getJobId(); Job job = context.getJob(jobID); TaskType type = taskID.getTaskType(); DataStatistics statistics = dataStatisticsForTask(taskID); int completedTasksOfType = type == TaskType.MAP ? job.getCompletedMaps() : job.getCompletedReduces(); int totalTasksOfType = type == TaskType.MAP ? job.getTotalMaps() : job.getTotalReduces(); if (completedTasksOfType < MINIMUM_COMPLETE_NUMBER_TO_SPECULATE || (((float)completedTasksOfType) / totalTasksOfType) < MINIMUM_COMPLETE_PROPORTION_TO_SPECULATE ) { return Long.MAX_VALUE; } long result = statistics == null ? Long.MAX_VALUE : (long)statistics.outlier(slowTaskRelativeTresholds.get(job)); return result; }
this.mapsCompleted = job.getCompletedMaps(); this.reducesTotal = job.getTotalReduces(); this.reducesCompleted = job.getCompletedReduces();
this.mapsCompleted = job.getCompletedMaps(); this.reducesTotal = job.getTotalReduces(); this.reducesCompleted = job.getCompletedReduces();
this.mapsCompleted = job.getCompletedMaps(); this.reducesTotal = job.getTotalReduces(); this.reducesCompleted = job.getCompletedReduces();
@Override public long thresholdRuntime(TaskId taskID) { JobId jobID = taskID.getJobId(); Job job = context.getJob(jobID); TaskType type = taskID.getTaskType(); DataStatistics statistics = dataStatisticsForTask(taskID); int completedTasksOfType = type == TaskType.MAP ? job.getCompletedMaps() : job.getCompletedReduces(); int totalTasksOfType = type == TaskType.MAP ? job.getTotalMaps() : job.getTotalReduces(); if (completedTasksOfType < MINIMUM_COMPLETE_NUMBER_TO_SPECULATE || (((float)completedTasksOfType) / totalTasksOfType) < MINIMUM_COMPLETE_PROPORTION_TO_SPECULATE ) { return Long.MAX_VALUE; } long result = statistics == null ? Long.MAX_VALUE : (long)statistics.outlier(slowTaskRelativeTresholds.get(job)); return result; }
this.state = job.getState(); this.mapsTotal = job.getTotalMaps(); this.mapsCompleted = job.getCompletedMaps(); this.mapProgress = report.getMapProgress() * 100; this.mapProgressPercent = percent(report.getMapProgress());
this.state = job.getState(); this.mapsTotal = job.getTotalMaps(); this.mapsCompleted = job.getCompletedMaps(); this.mapProgress = report.getMapProgress() * 100; this.mapProgressPercent =
this.state = job.getState(); this.mapsTotal = job.getTotalMaps(); this.mapsCompleted = job.getCompletedMaps(); this.mapProgress = report.getMapProgress() * 100; this.mapProgressPercent =
doReturn(10).when(job).getTotalMaps(); doReturn(10).when(job).getTotalReduces(); doReturn(0).when(job).getCompletedMaps(); RecalculateContainerAllocator allocator = new RecalculateContainerAllocator(rm, conf, appAttemptId, job); allocator.recalculatedReduceSchedule); doReturn(1).when(job).getCompletedMaps(); allocator.schedule(); Assert.assertTrue("Expected recalculate of reduce schedule",
public void verifyAMJobGeneric(Job job, String id, String user, String name, String state, long startTime, long finishTime, long elapsedTime, int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted, float reduceProgress, float mapProgress) { JobReport report = job.getReport(); WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()), id); WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(), user); WebServicesTestUtils.checkStringMatch("name", job.getName(), name); WebServicesTestUtils.checkStringMatch("state", job.getState().toString(), state); assertEquals("startTime incorrect", report.getStartTime(), startTime); assertEquals("finishTime incorrect", report.getFinishTime(), finishTime); assertEquals("elapsedTime incorrect", Times.elapsed(report.getStartTime(), report.getFinishTime()), elapsedTime); assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal); assertEquals("mapsCompleted incorrect", job.getCompletedMaps(), mapsCompleted); assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal); assertEquals("reducesCompleted incorrect", job.getCompletedReduces(), reducesCompleted); assertEquals("mapProgress incorrect", report.getMapProgress() * 100, mapProgress, 0); assertEquals("reduceProgress incorrect", report.getReduceProgress() * 100, reduceProgress, 0); }
Job job = context.getJob(jobId); JobUnsuccessfulCompletionEvent juce = (JobUnsuccessfulCompletionEvent) event; int successfulMaps = job.getCompletedMaps() - job.getFailedMaps() - job.getKilledMaps(); int successfulReduces = job.getCompletedReduces()