public static JobletProfile create(DataInput dis) throws IOException { JobletProfile jobletProfile = new JobletProfile(); jobletProfile.readFields(dis); return jobletProfile; }
@Override public void readFields(DataInput input) throws IOException { jobId = JobId.create(input); int size = input.readInt(); jobletProfiles = new HashMap<>(); for (int i = 0; i < size; i++) { String key = input.readUTF(); JobletProfile value = JobletProfile.create(input); jobletProfiles.put(key, value); } }
@Override protected void performEvent(TaskAttempt ta) { IJobManager jobManager = ccs.getJobManager(); JobRun run = jobManager.get(jobId); if (run == null) { return; } if (statistics != null) { JobProfile jobProfile = run.getJobProfile(); Map<String, JobletProfile> jobletProfiles = jobProfile.getJobletProfiles(); JobletProfile jobletProfile = jobletProfiles.get(nodeId); if (jobletProfile == null) { jobletProfile = new JobletProfile(nodeId); jobletProfiles.put(nodeId, jobletProfile); } jobletProfile.getTaskProfiles().put(taId, statistics); } run.getExecutor().notifyTaskComplete(ta); }
public void dumpProfile(JobletProfile jProfile) { Map<String, Long> counters = jProfile.getCounters(); counterMap.forEach((key, value) -> counters.put(key, value.get())); for (Task task : taskMap.values()) { TaskProfile taskProfile = new TaskProfile(task.getTaskAttemptId(), new Hashtable<>(task.getPartitionSendProfile()), new StatsCollector()); task.dumpProfile(taskProfile); jProfile.getTaskProfiles().put(task.getTaskAttemptId(), taskProfile); } }
@Override protected void doRun() throws Exception { List<JobProfile> profiles = new ArrayList<>(); Map<JobId, Joblet> jobletMap = ncs.getJobletMap(); jobletMap.values().stream().filter(ji -> ji.getJobId().getCcId().equals(ccId)) .forEach(ji -> profiles.add(new JobProfile(ji.getJobId()))); for (JobProfile jProfile : profiles) { Joblet ji; JobletProfile jobletProfile = new JobletProfile(ncs.getId()); ji = jobletMap.get(jProfile.getJobId()); if (ji != null) { ji.dumpProfile(jobletProfile); jProfile.getJobletProfiles().put(ncs.getId(), jobletProfile); } } fv.setValue(profiles); } }
private void updateJobStats(JobId jobId, Stats stats) { final IJobManager jobManager = ((ClusterControllerService) appCtx.getServiceContext().getControllerService()).getJobManager(); final JobRun run = jobManager.get(jobId); if (run == null || run.getStatus() != JobStatus.TERMINATED) { return; } final JobProfile jobProfile = run.getJobProfile(); final Collection<JobletProfile> jobletProfiles = jobProfile.getJobletProfiles().values(); long processedObjects = 0; for (JobletProfile jp : jobletProfiles) { final Collection<TaskProfile> jobletTasksProfile = jp.getTaskProfiles().values(); for (TaskProfile tp : jobletTasksProfile) { processedObjects += tp.getStatsCollector().getAggregatedStats().getTupleCounter().get(); } } stats.setProcessedObjects(processedObjects); }
@Override public ObjectNode toJSON() { ObjectMapper om = new ObjectMapper(); ObjectNode json = om.createObjectNode(); json.put("node-id", nodeId); populateCounters(json); ArrayNode tasks = om.createArrayNode(); for (TaskProfile p : taskProfiles.values()) { tasks.add(p.toJSON()); } json.set("tasks", tasks); return json; }
@Override public ObjectNode toJSON() { ObjectMapper om = new ObjectMapper(); ObjectNode json = om.createObjectNode(); json.put("job-id", jobId.toString()); populateCounters(json); ArrayNode jobletsArray = om.createArrayNode(); for (JobletProfile p : jobletProfiles.values()) { jobletsArray.add(p.toJSON()); } json.set("joblets", jobletsArray); return json; }