@GET @Produces(MediaType.APPLICATION_JSON) @Path("taskDistribution/{counterName}") public MRTaskExecutionResponse.TaskDistributionResponse getTaskDistributionByCounterName(@QueryParam("site") String site, @QueryParam("jobId") String jobId, @QueryParam("jobStartTime") String jobStartTime, @QueryParam("jobEndTime") String jobEndTime, @QueryParam("taskType") String taskType, @PathParam("counterName") String counterName, @QueryParam("distRange") String distRange) { MRTaskExecutionResponse.TaskDistributionResponse result = new MRTaskExecutionResponse.TaskDistributionResponse(); String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\" AND @taskType=\"%s\"]{@jobCounters}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, jobId, Constants.TaskType.MAP.toString()); GenericServiceAPIResponseEntity<TaskExecutionAPIEntity> response = ResourceUtils.getQueryResult(query, jobStartTime, jobEndTime); if (!response.isSuccess() || response.getObj() == null) { result.errMessage = response.getException(); return result; } try { return taskCountImpl.getHistoryTaskDistribution(response.getObj(), counterName, distRange); } catch (Exception e) { e.printStackTrace(); result.errMessage = e.getMessage(); return result; } }
if (taskType.equals(Constants.TaskType.MAP.toString()) && duration > jobExecutionEntity.getLastMapDuration()) { jobExecutionEntity.setLastMapDuration(duration); if (taskType.equals(Constants.TaskType.REDUCE.toString()) && duration > jobExecutionEntity.getLastReduceDuration()) { jobExecutionEntity.setLastReduceDuration(duration); if (taskType.equals(Constants.TaskType.MAP.toString()) && entity.getDuration() > jobExecutionEntity.getMaxMapTaskDuration()) { jobExecutionEntity.setMaxMapTaskDuration(entity.getDuration()); if (taskType.equals(Constants.TaskType.REDUCE.toString()) && entity.getDuration() > jobExecutionEntity.getMaxReduceTaskDuration()) { jobExecutionEntity.setMaxReduceTaskDuration(entity.getDuration()); if (taskType.equals(Constants.TaskType.MAP.toString())) { this.sumMapTaskDuration += entity.getDuration(); if (entity.getTaskStatus().equals(EagleTaskStatus.FAILED.name())
if (task.getTags().get(TASK_TYPE.toString()).equalsIgnoreCase(Constants.TaskType.MAP.toString())) { totalMapTime += task.getDuration(); if (mapStartTime > task.getStartTime()) {
if (Constants.TaskType.MAP.toString().equalsIgnoreCase(taskType)) { long mapTime = attempt.getEndTime() - attempt.getStartTime(); avgMapTimeInSec += mapTime;
private MRTaskExecutionResponse.TaskGroupResponse getTaskGroups(@QueryParam("site") String site, @QueryParam("shortJob_id") String shortDurationJobId, @QueryParam("longJob_id") String longDurationJobId) { MRTaskExecutionResponse.TaskGroupResponse result = new MRTaskExecutionResponse.TaskGroupResponse(); String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, shortDurationJobId); GenericServiceAPIResponseEntity<TaskExecutionAPIEntity> smallResponse = ResourceUtils.getQueryResult(query, null, null); if (!smallResponse.isSuccess() || smallResponse.getObj() == null) { result.errMessage = smallResponse.getException(); return result; } long longestDuration = 0; for (TaskExecutionAPIEntity entity : smallResponse.getObj()) { if (entity.getDuration() > longestDuration) { longestDuration = entity.getDuration(); } } query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, longDurationJobId); GenericServiceAPIResponseEntity<TaskExecutionAPIEntity> largeResponse = ResourceUtils.getQueryResult(query, null, null); if (!largeResponse.isSuccess() || largeResponse.getObj() == null) { result.errMessage = largeResponse.getException(); return result; } result.tasksGroupByType = new HashMap<>(); result.tasksGroupByType.put(Constants.TaskType.MAP.toString(), new MRTaskExecutionResponse.TaskGroup()); result.tasksGroupByType.put(Constants.TaskType.REDUCE.toString(), new MRTaskExecutionResponse.TaskGroup()); groupTasksByValue(result, false, largeResponse.getObj(), longestDuration); groupTasksByValue(result, true, smallResponse.getObj(), longestDuration); return result; }
private void taskAttemptEntityCreated(TaskAttemptExecutionAPIEntity entity) { JobCounters jobCounters = entity.getJobCounters(); String taskType = entity.getTags().get(TASK_TYPE.toString()); if (taskType != null && jobCounters != null && jobCounters.getCounters() != null) { if (Constants.TaskType.MAP.toString().equals(taskType.toUpperCase())) { mapAttemptDuration += entity.getDuration(); this.mapTaskAttemptCounterAgg.accumulate(jobCounters.getCounters().get(Constants.TASK_COUNTER)); this.mapFileSystemCounterAgg.accumulate(jobCounters.getCounters().get(Constants.FILE_SYSTEM_COUNTER)); return; } else if (Constants.TaskType.REDUCE.toString().equals(taskType.toUpperCase())) { reduceAttemptDuration += entity.getDuration(); this.reduceTaskAttemptCounterAgg.accumulate(jobCounters.getCounters().get(Constants.TASK_COUNTER)); this.reduceFileSystemTaskCounterAgg.accumulate(jobCounters.getCounters().get(Constants.FILE_SYSTEM_COUNTER)); return; } } ObjectMapper objectMapper = new ObjectMapper(); try { LOG.warn("Unknown task type of task attempt execution entity: " + objectMapper.writeValueAsString(entity)); } catch (Exception e) { LOG.error(e.getMessage(), e); } }
private MRTaskExecutionResponse.TaskGroup getTasks(TaskGroupResponse data) { return data.tasksGroupByType.get(Constants.TaskType.MAP.toString()); }
public void countTask(MRJobTaskCountResponse.UnitTaskCount counter, String taskType) { counter.taskCount++; if (taskType.equalsIgnoreCase(Constants.TaskType.MAP.toString())) { counter.mapTaskCount++; } else if (taskType.equalsIgnoreCase(Constants.TaskType.REDUCE.toString())) { counter.reduceTaskCount++; } }
@Override protected MRTaskExecutionResponse.TaskGroup getTasks(MRTaskExecutionResponse.TaskGroupResponse tasks) { return tasks.tasksGroupByType.get(Constants.TaskType.REDUCE.toString()); } }
@Override protected MRTaskExecutionResponse.TaskGroup getTasks(MRTaskExecutionResponse.TaskGroupResponse tasks) { return tasks.tasksGroupByType.get(Constants.TaskType.MAP.toString()); }
@Override protected MRTaskExecutionResponse.TaskGroup getTasks(MRTaskExecutionResponse.TaskGroupResponse tasks) { return tasks.tasksGroupByType.get(Constants.TaskType.MAP.toString()); } }
@Override protected MRTaskExecutionResponse.TaskGroup getTasks(MRTaskExecutionResponse.TaskGroupResponse tasks) { return tasks.tasksGroupByType.get(Constants.TaskType.REDUCE.toString()); } }