public List<String> getSearchTimeDuration(List<JobExecutionAPIEntity> jobEntities) { List<String> pair = new ArrayList<>(); long minStartTime = System.currentTimeMillis(); long maxEndTime = 0; for (JobExecutionAPIEntity jobEntity : jobEntities) { if (minStartTime > jobEntity.getStartTime()) { minStartTime = jobEntity.getStartTime(); } if (maxEndTime < jobEntity.getEndTime()) { maxEndTime = jobEntity.getEndTime(); } } pair.add(DateTimeUtil.millisecondsToHumanDateWithSeconds(minStartTime)); pair.add(DateTimeUtil.millisecondsToHumanDateWithSeconds(maxEndTime)); return pair; } }
@Override public void close() throws IOException { // check if this job history file is complete if (jobExecutionEntity.getEndTime() == 0L) { throw new IOException(new JHFWriteNotCompletedException(jobId)); } try { flush(); this.jobCounterMetricsGenerator.flush(); } catch (Exception ex) { throw new IOException(ex); } }
@Override public void flush(JobExecutionAPIEntity entity) { Map<String, Object> fields = new HashMap<>(entity.getTags()); fields.put("submissionTime", entity.getSubmissionTime()); fields.put("startTime", entity.getStartTime()); fields.put("endTime", entity.getEndTime()); fields.put("currentState", entity.getCurrentState()); fields.put("trackingUrl", entity.getTrackingUrl()); collector.collect(stormStreamId, new ValuesArray(fields.get(MRJobTagName.JOB_ID.toString()), fields)); } }
public JobCountResponse getRunningJobCount(List<JobExecutionAPIEntity> historyJobs, List<org.apache.eagle.jpm.mr.runningentity.JobExecutionAPIEntity> runningJobs, long startTimeInSecs, long endTimeInSecs, long intervalInSecs) { List<UnitJobCount> jobCounts = new ArrayList<>(); Set<String> jobTypes = new HashSet<>(); Set<String> jobIds = new HashSet<>(); initJobCountList(jobCounts, startTimeInSecs, endTimeInSecs, intervalInSecs); long startTimeInMs = startTimeInSecs * DateTimeUtil.ONESECOND; for (JobExecutionAPIEntity job: historyJobs) { jobIds.add(job.getTags().get(MRJobTagName.JOB_ID.toString())); if (job.getEndTime() >= startTimeInMs) { String jobType = job.getTags().get(MRJobTagName.JOB_TYPE.toString()); jobTypes.add(jobType); countJob(jobCounts, job.getStartTime() / DateTimeUtil.ONESECOND, job.getEndTime() / DateTimeUtil.ONESECOND, intervalInSecs, jobType); } } for (org.apache.eagle.jpm.mr.runningentity.JobExecutionAPIEntity job : runningJobs) { if (!ResourceUtils.isDuplicate(jobIds, job.getTags().get(MRJobTagName.JOB_ID.toString()))) { String jobType = job.getTags().get(MRJobTagName.JOB_TYPE.toString()); jobTypes.add(jobType); countJob(jobCounts, job.getStartTime() / DateTimeUtil.ONESECOND, endTimeInSecs, intervalInSecs, jobType); } } JobCountResponse response = new JobCountResponse(); response.jobCounts = jobCounts; response.jobTypes = jobTypes; return response; }
long maxFinishedTime = DateTimeUtil.humanDateToSeconds(endTime) * DateTimeUtil.ONESECOND; for (JobExecutionAPIEntity o : res.getObj()) { if (o.getEndTime() <= maxFinishedTime) { finishedJobs.add(o); jobIds.add(o.getTags().get(MRJobTagName.JOB_ID.toString()));
jobExecutionEntity.setStartTime(jobLaunchEventEntity.getTimestamp()); jobExecutionEntity.setEndTime(jobFinishEventEntity.getTimestamp()); jobExecutionEntity.setDurationTime(jobExecutionEntity.getEndTime() - jobExecutionEntity.getStartTime()); jobExecutionEntity.setTimestamp(jobLaunchEventEntity.getTimestamp()); jobExecutionEntity.setSubmissionTime(jobSubmitEventEntity.getTimestamp());
@Override public void jobEntityCreated(JobBaseAPIEntity entity) throws Exception { if (entity instanceof TaskExecutionAPIEntity) { info.getTasksMap().put(entity.getTags().get(TASK_ID.toString()), (TaskExecutionAPIEntity) entity); } else if (entity instanceof TaskAttemptExecutionAPIEntity) { info.getCompletedTaskAttemptsMap().put(entity.getTags().get(TASK_ATTEMPT_ID.toString()), (TaskAttemptExecutionAPIEntity) entity); } else if (entity instanceof JobExecutionAPIEntity) { JobExecutionAPIEntity jobExecutionAPIEntity = (JobExecutionAPIEntity) entity; info.setCurrentState(jobExecutionAPIEntity.getCurrentState()); info.setStartTime(jobExecutionAPIEntity.getStartTime()); info.setEndTime(jobExecutionAPIEntity.getEndTime()); info.setDurationTime(jobExecutionAPIEntity.getDurationTime()); info.setUserId(jobExecutionAPIEntity.getTags().get(MRJobTagName.USER.toString())); info.setJobId(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_ID.toString())); info.setJobDefId(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOD_DEF_ID.toString())); info.setSiteId(jobExecutionAPIEntity.getTags().get(MRJobTagName.SITE.toString())); info.setJobName(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_NAME.toString())) ; info.setJobQueueName(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_QUEUE.toString())); info.setJobType(jobExecutionAPIEntity.getTags().get(MRJobTagName.JOB_TYPE.toString())); info.setFinishedMaps(jobExecutionAPIEntity.getNumFinishedMaps()); info.setFinishedReduces(jobExecutionAPIEntity.getNumFinishedReduces()); info.setFailedReduces(jobExecutionAPIEntity.getNumFailedReduces()); info.setFailedMaps(jobExecutionAPIEntity.getNumFailedMaps()); info.setTotalMaps(jobExecutionAPIEntity.getNumTotalMaps()); info.setTotalReduces(jobExecutionAPIEntity.getNumTotalReduces()); info.setProgress(100); info.setTrackingUrl(((JobExecutionAPIEntity) entity).getTrackingUrl()); } }