@Override protected void doRun() throws Exception { List<JobProfile> profiles = new ArrayList<>(); Map<JobId, Joblet> jobletMap = ncs.getJobletMap(); jobletMap.values().stream().filter(ji -> ji.getJobId().getCcId().equals(ccId)) .forEach(ji -> profiles.add(new JobProfile(ji.getJobId()))); for (JobProfile jProfile : profiles) { Joblet ji; JobletProfile jobletProfile = new JobletProfile(ncs.getId()); ji = jobletMap.get(jProfile.getJobId()); if (ji != null) { ji.dumpProfile(jobletProfile); jProfile.getJobletProfiles().put(ncs.getId(), jobletProfile); } } fv.setValue(profiles); } }
@Override protected void performEvent(TaskAttempt ta) { IJobManager jobManager = ccs.getJobManager(); JobRun run = jobManager.get(jobId); if (run == null) { return; } if (statistics != null) { JobProfile jobProfile = run.getJobProfile(); Map<String, JobletProfile> jobletProfiles = jobProfile.getJobletProfiles(); JobletProfile jobletProfile = jobletProfiles.get(nodeId); if (jobletProfile == null) { jobletProfile = new JobletProfile(nodeId); jobletProfiles.put(nodeId, jobletProfile); } jobletProfile.getTaskProfiles().put(taId, statistics); } run.getExecutor().notifyTaskComplete(ta); }
private void updateJobStats(JobId jobId, Stats stats) { final IJobManager jobManager = ((ClusterControllerService) appCtx.getServiceContext().getControllerService()).getJobManager(); final JobRun run = jobManager.get(jobId); if (run == null || run.getStatus() != JobStatus.TERMINATED) { return; } final JobProfile jobProfile = run.getJobProfile(); final Collection<JobletProfile> jobletProfiles = jobProfile.getJobletProfiles().values(); long processedObjects = 0; for (JobletProfile jp : jobletProfiles) { final Collection<TaskProfile> jobletTasksProfile = jp.getTaskProfiles().values(); for (TaskProfile tp : jobletTasksProfile) { processedObjects += tp.getStatsCollector().getAggregatedStats().getTupleCounter().get(); } } stats.setProcessedObjects(processedObjects); }