@Override public void flush(JobExecutionAPIEntity entity) { Map<String, Object> fields = new HashMap<>(entity.getTags()); fields.put("submissionTime", entity.getSubmissionTime()); fields.put("startTime", entity.getStartTime()); fields.put("endTime", entity.getEndTime()); fields.put("currentState", entity.getCurrentState()); fields.put("trackingUrl", entity.getTrackingUrl()); collector.collect(stormStreamId, new ValuesArray(fields.get(MRJobTagName.JOB_ID.toString()), fields)); } }
public JobCountResponse getHistoryJobCountGroupByDuration(List<JobExecutionAPIEntity> jobDurations, String timeList) { JobCountResponse response = new JobCountResponse(); List<UnitJobCount> jobCounts = new ArrayList<>(); Set<String> jobTypes = new HashSet<>(); List<Long> times = ResourceUtils.parseDistributionList(timeList); for (int i = 0; i < times.size(); i++) { jobCounts.add(new UnitJobCount(times.get(i))); } for (JobExecutionAPIEntity job : jobDurations) { int jobIndex = ResourceUtils.getDistributionPosition(times, job.getDurationTime() / DateTimeUtil.ONESECOND); UnitJobCount counter = jobCounts.get(jobIndex); String jobType = job.getTags().get(MRJobTagName.JOB_TYPE.toString()); jobTypes.add(jobType); countJob(counter, jobType); } response.jobCounts = jobCounts; response.jobTypes = jobTypes; return response; }
List<JobExecutionAPIEntity> jobs = new ArrayList<>(); for (JobExecutionAPIEntity o : historyRes.getObj()) { if (o.getTags().get(MRJobTagName.JOB_TYPE.toString()).equalsIgnoreCase(jobType)) { jobs.add(o);
public JobCountResponse getRunningJobCount(List<JobExecutionAPIEntity> historyJobs, List<org.apache.eagle.jpm.mr.runningentity.JobExecutionAPIEntity> runningJobs, long startTimeInSecs, long endTimeInSecs, long intervalInSecs) { List<UnitJobCount> jobCounts = new ArrayList<>(); Set<String> jobTypes = new HashSet<>(); Set<String> jobIds = new HashSet<>(); initJobCountList(jobCounts, startTimeInSecs, endTimeInSecs, intervalInSecs); long startTimeInMs = startTimeInSecs * DateTimeUtil.ONESECOND; for (JobExecutionAPIEntity job: historyJobs) { jobIds.add(job.getTags().get(MRJobTagName.JOB_ID.toString())); if (job.getEndTime() >= startTimeInMs) { String jobType = job.getTags().get(MRJobTagName.JOB_TYPE.toString()); jobTypes.add(jobType); countJob(jobCounts, job.getStartTime() / DateTimeUtil.ONESECOND, job.getEndTime() / DateTimeUtil.ONESECOND, intervalInSecs, jobType); } } for (org.apache.eagle.jpm.mr.runningentity.JobExecutionAPIEntity job : runningJobs) { if (!ResourceUtils.isDuplicate(jobIds, job.getTags().get(MRJobTagName.JOB_ID.toString()))) { String jobType = job.getTags().get(MRJobTagName.JOB_TYPE.toString()); jobTypes.add(jobType); countJob(jobCounts, job.getStartTime() / DateTimeUtil.ONESECOND, endTimeInSecs, intervalInSecs, jobType); } } JobCountResponse response = new JobCountResponse(); response.jobCounts = jobCounts; response.jobTypes = jobTypes; return response; }
List<org.apache.eagle.jpm.mr.historyentity.JobExecutionAPIEntity> jobs = getJobs(site, currentTime, queryTimeRange.f0(), queryTimeRange.f1()); Set<String> jobIds = new HashSet<>(); jobs.forEach(job -> jobIds.add(job.getTags().get(JOB_ID.toString())));
if (o.getEndTime() <= maxFinishedTime) { finishedJobs.add(o); jobIds.add(o.getTags().get(MRJobTagName.JOB_ID.toString()));
if (entity != null) { Long timeStamp = entity.getTimestamp(); Map<String, String> tags = entity.getTags(); metrics.add(metricWrapper(timeStamp, Constants.JOB_EXECUTION_TIME,
@Override public void jobEntityCreated(JobBaseAPIEntity entity) throws Exception { if (entity instanceof TaskExecutionAPIEntity) { info.getTasksMap().put(entity.getTags().get(TASK_ID.toString()), (TaskExecutionAPIEntity) entity); } else if (entity instanceof TaskAttemptExecutionAPIEntity) { info.getCompletedTaskAttemptsMap().put(entity.getTags().get(TASK_ATTEMPT_ID.toString()), (TaskAttemptExecutionAPIEntity) entity); } else if (entity instanceof JobExecutionAPIEntity) { JobExecutionAPIEntity jobExecutionAPIEntity = (JobExecutionAPIEntity) entity; info.setCurrentState(jobExecutionAPIEntity.getCurrentState()); info.setStartTime(jobExecutionAPIEntity.getStartTime()); info.setEndTime(jobExecutionAPIEntity.getEndTime()); info.setDurationTime(jobExecutionAPIEntity.getDurationTime()); info.setUserId(jobExecutionAPIEntity.getTags().get(MRJobTagName.USER.toString())); info.setJobId(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_ID.toString())); info.setJobDefId(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOD_DEF_ID.toString())); info.setSiteId(jobExecutionAPIEntity.getTags().get(MRJobTagName.SITE.toString())); info.setJobName(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_NAME.toString())) ; info.setJobQueueName(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_QUEUE.toString())); info.setJobType(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_TYPE.toString())); info.setFinishedMaps(jobExecutionAPIEntity.getNumFinishedMaps()); info.setFinishedReduces(jobExecutionAPIEntity.getNumFinishedReduces()); info.setFailedReduces(jobExecutionAPIEntity.getNumFailedReduces()); info.setFailedMaps(jobExecutionAPIEntity.getNumFailedMaps()); info.setTotalMaps(jobExecutionAPIEntity.getNumTotalMaps()); info.setTotalReduces(jobExecutionAPIEntity.getNumTotalReduces()); info.setProgress(100); info.setTrackingUrl(((JobExecutionAPIEntity) entity).getTrackingUrl()); } }
jobSubmitEventEntity.getTags().put(MRJobTagName.JOB_NAME.toString(), jobName); jobSubmitEventEntity.getTags().put(MRJobTagName.JOD_DEF_ID.toString(), this.jobDefId); jobExecutionEntity.getTags().put(MRJobTagName.JOB_TYPE.toString(), this.jobType); entityCreated(jobSubmitEventEntity); } else if (values.get(Keys.LAUNCH_TIME) != null) { // job launched jobExecutionEntity.getTags().put(MRJobTagName.USER.toString(), user); jobExecutionEntity.getTags().put(MRJobTagName.JOB_ID.toString(), jobId); jobExecutionEntity.getTags().put(MRJobTagName.JOB_NAME.toString(), jobName); jobExecutionEntity.getTags().put(MRJobTagName.JOD_DEF_ID.toString(), jobDefId); jobExecutionEntity.getTags().put(MRJobTagName.JOB_QUEUE.toString(), queueName); jobExecutionEntity.getTags().put(MRJobTagName.JOB_TYPE.toString(), this.jobType); jobExecutionEntity.setAvgReduceTaskDuration(this.sumReduceTaskDuration * 1.0 / numTotalReduces); this.jobCounterMetricsGenerator.setBaseTags(jobExecutionEntity.getTags()); jobErrorCategoryMappingAPIEntity.setTags(new HashMap<>(jobExecutionEntity.getTags())); jobErrorCategoryMappingAPIEntity.getTags().put(MRJobTagName.ERROR_CATEGORY.toString(), errorCategory); jobErrorCategoryMappingAPIEntity.setTimestamp(jobExecutionEntity.getTimestamp());