public synchronized JobProfile getJobProfile(JobID jobid) { JobInProgress job = jobs.get(jobid); if (job != null) { return job.getProfile(); } else { return completedJobStatusStore.readJobProfile(jobid); } } public synchronized JobStatus getJobStatus(JobID jobid) {
/** * @deprecated Use {@link #getJobProfile(org.apache.hadoop.mapreduce.JobID)} * instead */ @Deprecated public JobProfile getJobProfile(JobID jobid) { synchronized (this) { JobInProgress job = jobs.get(jobid); if (job != null) { // Safe to call JobInProgress.getProfile while holding the lock // on the JobTracker since it isn't a synchronized method return job.getProfile(); } } return completedJobStatusStore.readJobProfile(jobid); }
public JobProfile getJobProfile(JobID jobid) { synchronized (this) { JobInProgress job = jobs.get(jobid); if (job != null) { // Safe to call JobInProgress.getProfile while holding the lock // on the JobTracker since it isn't a synchronized method return job.getProfile(); } else { RetireJobInfo info = retireJobs.get(jobid); if (info != null) { return info.profile; } } } return completedJobStatusStore.readJobProfile(jobid); } public JobStatus getJobStatus(JobID jobid) {
/** * For a JobInProgress that is being submitted, check whether * queue that the job has been submitted to exists and is RUNNING. * @param job The JobInProgress object being submitted. * @throws IOException */ public void checkQueueValidity(JobInProgress job) throws IOException { String queue = job.getProfile().getQueueName(); if (!(queueManager.getLeafQueueNames().contains(queue))) { throw new IOException("Queue \"" + queue + "\" does not exist"); } // check if queue is RUNNING if (!queueManager.isRunning(queue)) { throw new IOException("Queue \"" + queue + "\" is not running"); } }
synchronized void addToCache(JobInProgress job) { RetireJobInfo info = new RetireJobInfo(job.getStatus(), job.getProfile(), job.getFinishTime(), job.getHistoryFile()); jobRetireInfoQ.add(info); jobIDStatusMap.put(info.status.getJobID(), info); if (jobRetireInfoQ.size() > retiredJobsCacheSize) { RetireJobInfo removed = jobRetireInfoQ.remove(); jobIDStatusMap.remove(removed.status.getJobID()); LOG.info("Retired job removed from cache " + removed.status.getJobID()); } }
private void checkAccess(JobInProgress job, QueueManager.QueueOperation oper, UserGroupInformation ugi) throws IOException { // get the queue String queue = job.getProfile().getQueueName(); if (!queueManager.hasAccess(queue, job, oper, ugi)) { throw new AccessControlException("User " + ugi.getUserName() + " cannot perform " + "operation " + oper + " on queue " + queue + ".\n Please run \"hadoop queue -showacls\" " + "command to find the queues you have access" + " to ."); } }
private void checkAccess(JobInProgress job, QueueManager.QueueOperation oper) throws IOException { // get the user group info UserGroupInformation ugi = UserGroupInformation.getCurrentUGI(); // get the queue String queue = job.getProfile().getQueueName(); if (!queueManager.hasAccess(queue, job, oper, ugi)) { throw new AccessControlException("User " + ugi.getUserName() + " cannot perform " + "operation " + oper + " on queue " + queue); } }
private synchronized JobStatus[] getJobStatus(Collection<JobInProgress> jips, boolean toComplete) { if (jips == null || jips.isEmpty()) { return new JobStatus[]{}; } ArrayList<JobStatus> jobStatusList = new ArrayList<JobStatus>(); for(JobInProgress jip : jips) { JobStatus status = jip.getStatus(); status.setStartTime(jip.getStartTime()); status.setUsername(jip.getProfile().getUser()); if (toComplete) { if (status.getRunState() == JobStatus.RUNNING || status.getRunState() == JobStatus.PREP) { jobStatusList.add(status); } }else { jobStatusList.add(status); } } return jobStatusList.toArray( new JobStatus[jobStatusList.size()]); }
private synchronized JobStatus[] getJobStatus(Collection<JobInProgress> jips, boolean toComplete) { if(jips == null || jips.isEmpty()) { return new JobStatus[]{}; } ArrayList<JobStatus> jobStatusList = new ArrayList<JobStatus>(); for(JobInProgress jip : jips) { JobStatus status = jip.getStatus(); status.setStartTime(jip.getStartTime()); status.setUsername(jip.getProfile().getUser()); if(toComplete) { if(status.getRunState() == JobStatus.RUNNING || status.getRunState() == JobStatus.PREP) { jobStatusList.add(status); } }else { jobStatusList.add(status); } } return (JobStatus[]) jobStatusList.toArray( new JobStatus[jobStatusList.size()]); }
private synchronized JobStatus[] getJobStatus(Collection<JobInProgress> jips, boolean toComplete) { if(jips == null || jips.isEmpty()) { return new JobStatus[]{}; } ArrayList<JobStatus> jobStatusList = new ArrayList<JobStatus>(); for(JobInProgress jip : jips) { JobStatus status = jip.getStatus(); status.setStartTime(jip.getStartTime()); status.setUsername(jip.getProfile().getUser()); if(toComplete) { if(status.getRunState() == JobStatus.RUNNING || status.getRunState() == JobStatus.PREP) { jobStatusList.add(status); } }else { jobStatusList.add(status); } } return (JobStatus[]) jobStatusList.toArray( new JobStatus[jobStatusList.size()]); }
job.cleanupLocalizedJobConf(job.getProfile().getJobID()); jobs.remove(job.getProfile().getJobID()); for (JobInProgressListener l : jobInProgressListeners) { l.jobRemoved(job); String jobUser = job.getProfile().getUser(); LOG.info("Retired job with id: '" + job.getProfile().getJobID() + "' of user '" + jobUser + "'");
/** * Adds a job to the jobtracker. Make sure that the checks are inplace before * adding a job. This is the core job submission logic * @param jobId The id for the job submitted which needs to be added */ private synchronized JobStatus addJob(JobID jobId, JobInProgress job) throws IOException { totalSubmissions++; synchronized (jobs) { synchronized (taskScheduler) { jobs.put(job.getProfile().getJobID(), job); for (JobInProgressListener listener : jobInProgressListeners) { listener.jobAdded(job); } } } myInstrumentation.submitJob(this.conf, jobId); return job.getStatus(); }
/** * Adds a job to the jobtracker. Make sure that the checks are inplace before * adding a job. This is the core job submission logic * @param jobId The id for the job submitted which needs to be added */ synchronized JobStatus addJob(JobID jobId, JobInProgress job) { totalSubmissions++; synchronized (jobs) { synchronized (taskScheduler) { jobs.put(job.getProfile().getJobID(), job); for (JobInProgressListener listener : jobInProgressListeners) { try { listener.jobAdded(job); } catch (IOException ioe) { LOG.warn("Failed to add and so skipping the job : " + job.getJobID() + ". Exception : " + ioe); } } } } myInstrumentation.submitJob(job.getJobConf(), jobId); LOG.info("Job " + jobId + " added successfully for user '" + job.getJobConf().getUser() + "' to queue '" + job.getJobConf().getQueueName() + "'"); return job.getStatus(); }
job.getProfile().write(dataOut);
job.getProfile().write(dataOut);
job.getProfile().write(dataOut);
jobs.put(job.getProfile().getJobID(), job); for (JobInProgressListener listener : jobInProgressListeners) { try {
/** * Check the ACLs for a user doing the passed operation. * <ul> * <li>If ACLs are disabled, allow all users.</li> * <li>Otherwise, if the operation is not a job operation(for eg. * submit-job-to-queue), then allow only (a) clusterOwner(who started the * cluster), (b) cluster administrators and (c) members of * queue-submit-job-acl for the queue.</li> * <li>If the operation is a job operation, then allow only (a) jobOwner, * (b) clusterOwner(who started the cluster), (c) cluster administrators, * (d) members of queue admins acl for the queue and (e) members of job * acl for the job operation</li> * </ul> * * @param job the job on which operation is requested * @param callerUGI the user who is requesting the operation * @param operation the operation for which authorization is needed * @throws AccessControlException */ void checkAccess(JobInProgress job, UserGroupInformation callerUGI, Operation operation) throws AccessControlException { String queue = job.getProfile().getQueueName(); String jobId = job.getJobID().toString(); JobStatus jobStatus = job.getStatus(); String jobOwner = jobStatus.getUsername(); AccessControlList jobAcl = jobStatus.getJobACLs().get(operation.jobACLNeeded); checkAccess(jobId, callerUGI, queue, operation, jobOwner, jobAcl); }
/** * Returns an XML-formatted table of the jobs in the list. * This is called repeatedly for different lists of jobs (e.g., running, completed, failed). */ public void generateJobTable(JspWriter out, String label, List<JobInProgress> jobs) throws IOException { if (jobs.size() > 0) { for (JobInProgress job : jobs) { JobProfile profile = job.getProfile(); JobStatus status = job.getStatus(); JobID jobid = profile.getJobID(); int desiredMaps = job.desiredMaps(); int desiredReduces = job.desiredReduces(); int completedMaps = job.finishedMaps(); int completedReduces = job.finishedReduces(); String name = profile.getJobName(); out.print("<" + label + "_job jobid=\"" + jobid + "\">\n"); out.print(" <jobid>" + jobid + "</jobid>\n"); out.print(" <user>" + profile.getUser() + "</user>\n"); out.print(" <name>" + ("".equals(name) ? " " : name) + "</name>\n"); out.print(" <map_complete>" + StringUtils.formatPercent(status.mapProgress(), 2) + "</map_complete>\n"); out.print(" <map_total>" + desiredMaps + "</map_total>\n"); out.print(" <maps_completed>" + completedMaps + "</maps_completed>\n"); out.print(" <reduce_complete>" + StringUtils.formatPercent(status.reduceProgress(), 2) + "</reduce_complete>\n"); out.print(" <reduce_total>" + desiredReduces + "</reduce_total>\n"); out.print(" <reduces_completed>" + completedReduces + "</reduces_completed>\n"); out.print("</" + label + "_job>\n"); } } }
/** * Returns an XML-formatted table of the jobs in the list. * This is called repeatedly for different lists of jobs (e.g., running, completed, failed). */ public void generateJobTable(JspWriter out, String label, List<JobInProgress> jobs) throws IOException { if (jobs.size() > 0) { for (JobInProgress job : jobs) { JobProfile profile = job.getProfile(); JobStatus status = job.getStatus(); JobID jobid = profile.getJobID(); int desiredMaps = job.desiredMaps(); int desiredReduces = job.desiredReduces(); int completedMaps = job.finishedMaps(); int completedReduces = job.finishedReduces(); String name = profile.getJobName(); out.print("<" + label + "_job jobid=\"" + jobid + "\">\n"); out.print(" <jobid>" + jobid + "</jobid>\n"); out.print(" <user>" + profile.getUser() + "</user>\n"); out.print(" <name>" + ("".equals(name) ? " " : name) + "</name>\n"); out.print(" <map_complete>" + StringUtils.formatPercent(status.mapProgress(), 2) + "</map_complete>\n"); out.print(" <map_total>" + desiredMaps + "</map_total>\n"); out.print(" <maps_completed>" + completedMaps + "</maps_completed>\n"); out.print(" <reduce_complete>" + StringUtils.formatPercent(status.reduceProgress(), 2) + "</reduce_complete>\n"); out.print(" <reduce_total>" + desiredReduces + "</reduce_total>\n"); out.print(" <reduces_completed>" + completedReduces + "</reduces_completed>\n"); out.print("</" + label + "_job>\n"); } } }