public void taskExecutionEntityCreated(TaskExecutionAPIEntity taskExecutionAPIEntity) { JobCounters jobCounters = taskExecutionAPIEntity.getJobCounters(); if (jobCounters == null || jobCounters.getCounters() == null) { LOG.warn("found null job counters, task {}", taskExecutionAPIEntity.getTags().get(MRJobTagName.TASK_ID.toString())); return; long duration = taskExecutionAPIEntity.getDuration(); long startTime = taskExecutionAPIEntity.getStartTime(); long endTime = taskExecutionAPIEntity.getEndTime();
public MRTaskExecutionResponse.TaskGroupResponse groupTasksByValue(MRTaskExecutionResponse.TaskGroupResponse result, boolean keepShort, List<TaskExecutionAPIEntity> tasks, long value) { for (TaskExecutionAPIEntity entity : tasks) { String taskType = entity.getTags().get(MRJobTagName.TASK_TYPE.toString()); MRTaskExecutionResponse.TaskGroup taskGroup = result.tasksGroupByType.get(taskType.toUpperCase()); if (entity.getDuration() <= value && keepShort) { taskGroup.shortTasks.add(entity); } if (entity.getDuration() > value) { taskGroup.longTasks.add(entity); } } return result; }
public MRJobTaskCountResponse.HistoryTaskCountResponse countHistoryTask(List<org.apache.eagle.jpm.mr.historyentity.TaskExecutionAPIEntity> tasks, long startTimeInMin, long endTimeInMin) { List<MRJobTaskCountResponse.UnitTaskCount> taskCounts = new ArrayList<>(); for (long i = startTimeInMin; i <= endTimeInMin; i++) { taskCounts.add(new MRJobTaskCountResponse.UnitTaskCount(i * DateTimeUtil.ONEMINUTE, null)); } for (org.apache.eagle.jpm.mr.historyentity.TaskExecutionAPIEntity task : tasks) { String taskType = task.getTags().get(MRJobTagName.TASK_TYPE.toString()); long taskStarTimeMin = task.getStartTime() / DateTimeUtil.ONEMINUTE; long taskEndTimeMin = task.getEndTime() / DateTimeUtil.ONEMINUTE; int relativeStartTime = (int) (taskStarTimeMin - startTimeInMin); int relativeEndTime = (int) (taskEndTimeMin - startTimeInMin); for (int i = relativeStartTime; i <= relativeEndTime; i++) { countTask(taskCounts.get(i), taskType); } } MRJobTaskCountResponse.HistoryTaskCountResponse response = new MRJobTaskCountResponse.HistoryTaskCountResponse(); response.taskCount = taskCounts; return response; }
taskExecutionTags.put(MRJobTagName.HOSTNAME.toString(), hostname); taskExecutionTags.put(MRJobTagName.RACK.toString(), host2RackMapping.get(hostname)); TaskExecutionAPIEntity entity = new TaskExecutionAPIEntity(); entity.setTags(taskExecutionTags); entity.setStartTime(taskStartTime.get(taskID)); entity.setEndTime(Long.valueOf(finishTime)); entity.setDuration(entity.getEndTime() - entity.getStartTime()); entity.setTimestamp(jobLaunchTime); entity.setError(values.get(Keys.ERROR)); entity.setTaskStatus(values.get(Keys.TASK_STATUS)); if (values.get(Keys.COUNTERS) != null || counters != null) { entity.setJobCounters(parseCounters(counters)); long duration = entity.getEndTime() - jobLaunchTime; if (taskType.equals(Constants.TaskType.MAP.toString()) && duration > jobExecutionEntity.getLastMapDuration()) { jobExecutionEntity.setLastMapDuration(duration); if (taskType.equals(Constants.TaskType.MAP.toString()) && entity.getDuration() > jobExecutionEntity.getMaxMapTaskDuration()) { jobExecutionEntity.setMaxMapTaskDuration(entity.getDuration()); if (taskType.equals(Constants.TaskType.REDUCE.toString()) && entity.getDuration() > jobExecutionEntity.getMaxReduceTaskDuration()) { jobExecutionEntity.setMaxReduceTaskDuration(entity.getDuration()); this.sumMapTaskDuration += entity.getDuration(); if (entity.getTaskStatus().equals(EagleTaskStatus.FAILED.name()) || entity.getTaskStatus().equals(EagleTaskStatus.KILLED.name())) { jobExecutionEntity.setNumFailedMaps(1 + jobExecutionEntity.getNumFailedMaps()); this.sumReduceTaskDuration += entity.getDuration();
@Override public int compare(org.apache.eagle.jpm.mr.historyentity.TaskExecutionAPIEntity o1, org.apache.eagle.jpm.mr.historyentity.TaskExecutionAPIEntity o2) { Long time1 = o1.getDuration(); Long time2 = o2.getDuration(); return (time1 > time2 ? -1 : (time1.equals(time2)) ? 0 : 1); } }
private double getAverageSpillBytes(List<TaskExecutionAPIEntity> tasks) { if (tasks.isEmpty()) { return 0; } long totalSpillBytes = 0; for (TaskExecutionAPIEntity task : tasks) { totalSpillBytes += task.getJobCounters().getCounterValue(JobCounters.CounterName.SPLIT_RAW_BYTES); } return totalSpillBytes / tasks.size(); }
taskCountImpl.initTaskCountList(runningTaskCount, finishedTaskCount, times, new MRTaskCountImpl.HistoryTaskComparator()); for (org.apache.eagle.jpm.mr.historyentity.TaskExecutionAPIEntity o : historyRes.getObj()) { int index = ResourceUtils.getDistributionPosition(times, o.getDuration() / DateTimeUtil.ONESECOND); MRJobTaskCountResponse.UnitTaskCount counter = finishedTaskCount.get(index); taskCountImpl.countTask(counter, o.getTags().get(TASK_TYPE.toString())); counter.entities.add(o);
private MRTaskExecutionResponse.TaskGroupResponse getTaskGroups(@QueryParam("site") String site, @QueryParam("shortJob_id") String shortDurationJobId, @QueryParam("longJob_id") String longDurationJobId) { MRTaskExecutionResponse.TaskGroupResponse result = new MRTaskExecutionResponse.TaskGroupResponse(); String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, shortDurationJobId); GenericServiceAPIResponseEntity<TaskExecutionAPIEntity> smallResponse = ResourceUtils.getQueryResult(query, null, null); if (!smallResponse.isSuccess() || smallResponse.getObj() == null) { result.errMessage = smallResponse.getException(); return result; } long longestDuration = 0; for (TaskExecutionAPIEntity entity : smallResponse.getObj()) { if (entity.getDuration() > longestDuration) { longestDuration = entity.getDuration(); } } query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, longDurationJobId); GenericServiceAPIResponseEntity<TaskExecutionAPIEntity> largeResponse = ResourceUtils.getQueryResult(query, null, null); if (!largeResponse.isSuccess() || largeResponse.getObj() == null) { result.errMessage = largeResponse.getException(); return result; } result.tasksGroupByType = new HashMap<>(); result.tasksGroupByType.put(Constants.TaskType.MAP.toString(), new MRTaskExecutionResponse.TaskGroup()); result.tasksGroupByType.put(Constants.TaskType.REDUCE.toString(), new MRTaskExecutionResponse.TaskGroup()); groupTasksByValue(result, false, largeResponse.getObj(), longestDuration); groupTasksByValue(result, true, smallResponse.getObj(), longestDuration); return result; }
public static double[] getCounterValues(List<TaskExecutionAPIEntity> tasks, JobCounters.CounterName counterName) { List<Double> values = new ArrayList<>(); for (TaskExecutionAPIEntity task : tasks) { values.add(Double.valueOf(task.getJobCounters().getCounterValue(counterName))); } return toArray(values); }
if (task.getTags().get(TASK_TYPE.toString()).equalsIgnoreCase(Constants.TaskType.MAP.toString())) { totalMapTime += task.getDuration(); if (mapStartTime > task.getStartTime()) { mapStartTime = task.getStartTime(); if (mapEndTime < task.getEndTime()) { mapEndTime = task.getEndTime(); totalReduceTime += task.getDuration(); if (reduceStartTime > task.getStartTime()) { reduceStartTime = task.getStartTime(); if (reduceEndTime < task.getEndTime()) { reduceEndTime = task.getEndTime();
public MRTaskExecutionResponse.TaskDistributionResponse getHistoryTaskDistribution(List<org.apache.eagle.jpm.mr.historyentity.TaskExecutionAPIEntity> tasks, String counterName, String distRange) { MRTaskExecutionResponse.TaskDistributionResponse response = new MRTaskExecutionResponse.TaskDistributionResponse(); response.counterName = counterName; List<Long> distRangeList = ResourceUtils.parseDistributionList(distRange); for (int i = 0; i < distRangeList.size(); i++) { response.taskBuckets.add(new MRTaskExecutionResponse.CountUnit(distRangeList.get(i))); } JobCounters.CounterName jobCounterName = JobCounters.CounterName.valueOf(counterName.toUpperCase()); for (org.apache.eagle.jpm.mr.historyentity.TaskExecutionAPIEntity task : tasks) { Long counterValue = task.getJobCounters().getCounterValue(jobCounterName); int pos = ResourceUtils.getDistributionPosition(distRangeList, counterValue); response.taskBuckets.get(pos).countVal++; } return response; }