@Override public GetTaskReportsResponse getTaskReports(GetTaskReportsRequest request) throws IOException { JobId jobId = request.getJobId(); TaskType taskType = request.getTaskType(); GetTaskReportsResponse response = recordFactory.newRecordInstance(GetTaskReportsResponse.class); Job job = verifyAndGetJob(jobId, true); Collection<Task> tasks = job.getTasks(taskType).values(); for (Task task : tasks) { response.addTaskReport(task.getReport()); } return response; }
private void checkAccess(Job job, JobACL jobOperation) throws IOException { UserGroupInformation callerUGI; callerUGI = UserGroupInformation.getCurrentUser(); if (!job.checkAccess(callerUGI, jobOperation)) { throw new IOException(new AccessControlException("User " + callerUGI.getShortUserName() + " cannot perform operation " + jobOperation.name() + " on " + job.getID())); } }
@Override public GetCountersResponse getCounters(GetCountersRequest request) throws IOException { JobId jobId = request.getJobId(); Job job = verifyAndGetJob(jobId, true); GetCountersResponse response = recordFactory.newRecordInstance(GetCountersResponse.class); response.setCounters(TypeConverter.toYarn(job.getAllCounters())); return response; }
public JobInfo(Job job, Boolean hasAccess) { this.id = MRApps.toString(job.getID()); JobReport report = job.getReport(); this.startTime = report.getStartTime(); this.finishTime = report.getFinishTime(); this.elapsedTime = 0; this.name = job.getName().toString(); this.user = job.getUserName(); this.state = job.getState(); this.mapsTotal = job.getTotalMaps(); this.mapsCompleted = job.getCompletedMaps(); this.mapProgress = report.getMapProgress() * 100; this.mapProgressPercent = percent(report.getMapProgress()); this.reducesTotal = job.getTotalReduces(); this.reducesCompleted = job.getCompletedReduces(); this.reduceProgress = report.getReduceProgress() * 100; this.reduceProgressPercent = percent(report.getReduceProgress()); this.uberized = job.isUber(); List<String> diagnostics = job.getDiagnostics(); if (diagnostics != null && !diagnostics.isEmpty()) { StringBuffer b = new StringBuffer(); Map<JobACL, AccessControlList> allacls = job.getJobACLs(); if (allacls != null) { for (Map.Entry<JobACL, AccessControlList> entry : allacls.entrySet()) {
public void verifyAMJobGeneric(Job job, String id, String user, String name, String state, long startTime, long finishTime, long elapsedTime, int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted, float reduceProgress, float mapProgress) { JobReport report = job.getReport(); WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()), id); WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(), user); WebServicesTestUtils.checkStringMatch("name", job.getName(), name); WebServicesTestUtils.checkStringMatch("state", job.getState().toString(), state); assertEquals("startTime incorrect", report.getStartTime(), startTime); assertEquals("finishTime incorrect", report.getFinishTime(), finishTime); assertEquals("elapsedTime incorrect", Times.elapsed(report.getStartTime(), report.getFinishTime()), elapsedTime); assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal); assertEquals("mapsCompleted incorrect", job.getCompletedMaps(), mapsCompleted); assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal); assertEquals("reducesCompleted incorrect", job.getCompletedReduces(), reducesCompleted); assertEquals("mapProgress incorrect", report.getMapProgress() * 100, mapProgress, 0); assertEquals("reduceProgress incorrect", report.getReduceProgress() * 100, reduceProgress, 0); }
@GET @Path("/jobs/{jobid}/jobattempts") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public AMAttemptsInfo getJobAttempts(@PathParam("jobid") String jid) { init(); Job job = getJobFromJobIdString(jid, appCtx); AMAttemptsInfo amAttempts = new AMAttemptsInfo(); for (AMInfo amInfo : job.getAMInfos()) { AMAttemptInfo attempt = new AMAttemptInfo(amInfo, MRApps.toString( job.getID()), job.getUserName()); amAttempts.add(attempt); } return amAttempts; }
@Override public long thresholdRuntime(TaskId taskID) { JobId jobID = taskID.getJobId(); Job job = context.getJob(jobID); TaskType type = taskID.getTaskType(); DataStatistics statistics = dataStatisticsForTask(taskID); int completedTasksOfType = type == TaskType.MAP ? job.getCompletedMaps() : job.getCompletedReduces(); int totalTasksOfType = type == TaskType.MAP ? job.getTotalMaps() : job.getTotalReduces(); if (completedTasksOfType < MINIMUM_COMPLETE_NUMBER_TO_SPECULATE || (((float)completedTasksOfType) / totalTasksOfType) < MINIMUM_COMPLETE_PROPORTION_TO_SPECULATE ) { return Long.MAX_VALUE; } long result = statistics == null ? Long.MAX_VALUE : (long)statistics.outlier(slowTaskRelativeTresholds.get(job)); return result; }
private Job mockJob() { Job mockJob = mock(Job.class); when(mockJob.getAllCounters()).thenReturn(new Counters()); when(mockJob.getTotalMaps()).thenReturn(10); when(mockJob.getTotalReduces()).thenReturn(10); when(mockJob.getName()).thenReturn("mockjob"); return mockJob; }
@Override protected synchronized void heartbeat() throws Exception { scheduleStats.updateAndLogIfChanged("Before Scheduling: "); List<Container> allocatedContainers = getResources(); if (allocatedContainers != null && allocatedContainers.size() > 0) { scheduledRequests.assign(allocatedContainers); } int completedMaps = getJob().getCompletedMaps(); int completedTasks = completedMaps + getJob().getCompletedReduces(); if ((lastCompletedTasks != completedTasks) || (scheduledRequests.maps.size() > 0)) { lastCompletedTasks = completedTasks; recalculateReduceSchedule = true; } if (recalculateReduceSchedule) { boolean reducerPreempted = preemptReducesIfNeeded(); if (!reducerPreempted) { // Only schedule new reducers if no reducer preemption happens for // this heartbeat scheduleReduces(getJob().getTotalMaps(), completedMaps, scheduledRequests.maps.size(), scheduledRequests.reduces.size(), assignedRequests.maps.size(), assignedRequests.reduces.size(), mapResourceRequest, reduceResourceRequest, pendingReduces.size(), maxReduceRampupLimit, reduceSlowStart); } recalculateReduceSchedule = false; } scheduleStats.updateAndLogIfChanged("After Scheduling: "); }
when(job.getReport()).thenReturn( MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0, 0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); doReturn(10).when(job).getTotalMaps(); doReturn(10).when(job).getTotalReduces(); doReturn(0).when(job).getCompletedMaps(); RecalculateContainerAllocator allocator = new RecalculateContainerAllocator(rm, conf, appAttemptId, job); allocator.recalculatedReduceSchedule); doReturn(1).when(job).getCompletedMaps(); allocator.schedule(); Assert.assertTrue("Expected recalculate of reduce schedule",
private void printStat(Job job, long startTime) throws Exception { long currentTime = System.currentTimeMillis(); Runtime.getRuntime().gc(); long mem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); System.out.println("JobState:" + job.getState() + " CompletedMaps:" + job.getCompletedMaps() + " CompletedReduces:" + job.getCompletedReduces() + " Memory(total-free)(KB):" + mem/1024 + " ElapsedTime(ms):" + (currentTime - startTime)); }
task = job.getTask(taskID); if (task == null) { return; Map<TaskId, Task> tasks = job.getTasks(); total = job.getAllCounters(); boolean needTotalCounters = false; if (total == null) {
return; total = job.getAllCounters(); boolean needTotalCounters = false; if (total == null) { reduce = new Counters(); Map<TaskId, Task> tasks = job.getTasks(); for (Task t : tasks.values()) { Counters counters = t.getCounters();
@Override public GetTaskReportResponse getTaskReport(GetTaskReportRequest request) throws IOException { TaskId taskId = request.getTaskId(); Job job = verifyAndGetJob(taskId.getJobId(), true); GetTaskReportResponse response = recordFactory.newRecordInstance(GetTaskReportResponse.class); response.setTaskReport(job.getTask(taskId).getReport()); return response; }
@Override public GetTaskAttemptCompletionEventsResponse getTaskAttemptCompletionEvents( GetTaskAttemptCompletionEventsRequest request) throws IOException { JobId jobId = request.getJobId(); int fromEventId = request.getFromEventId(); int maxEvents = request.getMaxEvents(); Job job = verifyAndGetJob(jobId, true); GetTaskAttemptCompletionEventsResponse response = recordFactory.newRecordInstance(GetTaskAttemptCompletionEventsResponse.class); response.addAllCompletionEvents(Arrays.asList(job.getTaskAttemptCompletionEvents(fromEventId, maxEvents))); return response; }
Job job = app.submit(conf); app.waitForState(job, JobState.RUNNING); long jobStartTime = job.getReport().getStartTime(); 3, job.getTasks().size()); Iterator<Task> it = job.getTasks().values().iterator(); Task mapTask1 = it.next(); Task mapTask2 = it.next(); 3, job.getTasks().size()); it = job.getTasks().values().iterator(); mapTask1 = it.next(); mapTask2 = it.next(); app.verifyCompleted(); Assert.assertEquals("Job Start time not correct", jobStartTime, job.getReport().getStartTime()); Assert.assertEquals("Task Start time not correct", task1StartTime, mapTask1.getReport().getStartTime()); Assert.assertEquals("Task Finish time not correct", task1FinishTime, mapTask1.getReport().getFinishTime()); Assert.assertEquals(2, job.getAMInfos().size()); int attemptNum = 1; for (AMInfo amInfo : job.getAMInfos()) { Assert.assertEquals(attemptNum++, amInfo.getAppAttemptId() .getAttemptId());
public void updateAndLogIfChanged(String msgPrefix) { boolean changed = false; // synchronized to fix findbug warnings synchronized (RMContainerAllocator.this) { changed |= (numPendingReduces != pendingReduces.size()); numPendingReduces = pendingReduces.size(); changed |= (numScheduledMaps != scheduledRequests.maps.size()); numScheduledMaps = scheduledRequests.maps.size(); changed |= (numScheduledReduces != scheduledRequests.reduces.size()); numScheduledReduces = scheduledRequests.reduces.size(); changed |= (numAssignedMaps != assignedRequests.maps.size()); numAssignedMaps = assignedRequests.maps.size(); changed |= (numAssignedReduces != assignedRequests.reduces.size()); numAssignedReduces = assignedRequests.reduces.size(); changed |= (numCompletedMaps != getJob().getCompletedMaps()); numCompletedMaps = getJob().getCompletedMaps(); changed |= (numCompletedReduces != getJob().getCompletedReduces()); numCompletedReduces = getJob().getCompletedReduces(); changed |= (numContainersAllocated != containersAllocated); numContainersAllocated = containersAllocated; changed |= (numContainersReleased != containersReleased); numContainersReleased = containersReleased; } if (changed) { log(msgPrefix); } }
@Test public void testSingleCounterView() { AppContext appContext = new MockAppContext(0, 1, 1, 1); Job job = appContext.getAllJobs().values().iterator().next(); // add a failed task to the job without any counters Task failedTask = MockJobs.newTask(job.getID(), 2, 1, true); Map<TaskId,Task> tasks = job.getTasks(); tasks.put(failedTask.getID(), failedTask); Map<String, String> params = getJobParams(appContext); params.put(AMParams.COUNTER_GROUP, "org.apache.hadoop.mapreduce.FileSystemCounter"); params.put(AMParams.COUNTER_NAME, "HDFS_WRITE_OPS"); WebAppTests.testPage(SingleCounterPage.class, AppContext.class, appContext, params); }
@Override public GetJobReportResponse getJobReport(GetJobReportRequest request) throws IOException { JobId jobId = request.getJobId(); Job job = verifyAndGetJob(jobId, false); GetJobReportResponse response = recordFactory.newRecordInstance(GetJobReportResponse.class); if (job != null) { response.setJobReport(job.getReport()); } else { response.setJobReport(null); } return response; }
private boolean hasAccess(Job job, HttpServletRequest request) { String remoteUser = request.getRemoteUser(); if (remoteUser != null) { return job.checkAccess(UserGroupInformation.createRemoteUser(remoteUser), JobACL.VIEW_JOB); } return true; }