@Override public void flush(TaskAttemptExecutionAPIEntity entity) { Map<String, Object> fields = new HashMap<>(entity.getTags()); fields.put("startTime", entity.getStartTime()); fields.put("endTime", entity.getEndTime()); fields.put("taskStatus", entity.getTaskStatus()); if (!fields.containsKey(MRJobTagName.ERROR_CATEGORY.toString())) { fields.put("errorCategory", ""); } collector.collect(stormStreamId, new ValuesArray(fields.get(MRJobTagName.TASK_ATTEMPT_ID.toString()), fields)); } }
String taskType = getTaskType(attempt); if (Constants.TaskType.MAP.toString().equalsIgnoreCase(taskType)) { long mapTime = attempt.getEndTime() - attempt.getStartTime(); avgMapTimeInSec += mapTime; if (firstMap == null || firstMap.getStartTime() > attempt.getStartTime()) { firstMap = attempt; lastMap = attempt; if (worstMap == null || (worstMap.getEndTime() - worstMap.getStartTime()) < mapTime) { worstMap = attempt; long shuffleTime = attempt.getShuffleFinishTime() - attempt.getStartTime(); avgShuffleTimeInSec += shuffleTime; if (firstShuffle == null || firstShuffle.getStartTime() > attempt.getStartTime()) { firstShuffle = attempt; lastShuffle = attempt; if (worstShuffle == null || (worstShuffle.getShuffleFinishTime() - worstShuffle.getStartTime()) < shuffleTime) { worstShuffle = attempt; if (firstReduce == null || firstReduce.getStartTime() > attempt.getStartTime()) { firstReduce = attempt;
long mapPhaseTimeInSec = (lastMap.getEndTime() - firstMap.getStartTime()) / DateTimeUtil.ONESECOND; if (mapPhaseTimeInSec > context.getAvgMapTimeInSec() * ((context.getNumMaps() + tasksPerTime - 1) / tasksPerTime) * 20) { long reducePhaseTimeInSec = (lastReduce.getEndTime() - firstShuffle.getStartTime()) / DateTimeUtil.ONESECOND; if (reducePhaseTimeInSec > context.getAvgReduceTimeInSec() * ((context.getNumReduces() + tasksPerTime - 1) / tasksPerTime) * 20) {
entity.setEndTime(Long.valueOf(finishTime)); entity.setTimestamp(jobLaunchTime); entity.setDuration(entity.getEndTime() - entity.getStartTime()); entity.setTaskStatus(values.get(Keys.TASK_STATUS)); entity.setError(values.get(Keys.ERROR)); taskAttemptErrorCategoryEntity.setTags(taskAttemptErrorCategoryEntityTags); taskAttemptErrorCategoryEntity.setStartTime(entity.getStartTime()); taskAttemptErrorCategoryEntity.setEndTime(entity.getEndTime()); taskAttemptErrorCategoryEntity.setTimestamp(entity.getTimestamp());