/** * Returns the tracking ID of this job. * @param context current execution context * @return the tracking ID * @since 0.5.0 */ public String getTrackingId(ExecutionContext context) { return computeTrackingId(context); }
private void done(Job job) { assert job != null; blockers.remove(job.getId()); } }
private boolean isBlocked(Job job) { assert job != null; for (String blocker : job.getBlockerIds()) { if (blockers.contains(blocker)) { return true; } } return false; }
private static Callable<Void> build(ExecutionMonitor monitor, ExecutionContext context, Job job) { if (monitor == null) { throw new IllegalArgumentException("monitor must not be null"); //$NON-NLS-1$ } if (context == null) { throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$ } if (job == null) { throw new IllegalArgumentException("job must not be null"); //$NON-NLS-1$ } return () -> { LOG.debug("Starting job: {} in {}", job.getId(), Thread.currentThread().getName()); job.launch(monitor, context); LOG.debug("Completing job: {} in {}", job.getId(), Thread.currentThread().getName()); return null; }; }
throw new IllegalArgumentException("job must not be null"); //$NON-NLS-1$ String resourceId = job.getResourceId(context); ExecutorService executor = resourceExecutors.get(resourceId); if (executor == null) { context.getExecutionId(), context.getPhase(), job.getJobLabel(), job.getServiceLabel(), resourceId); LOG.debug("Resource {} is not defined: {}", resourceId, job.getId()); executor = defaultExecutor; } else { context.getExecutionId(), context.getPhase(), job.getJobLabel(), job.getServiceLabel(), resourceId);
getJobLabel(), getServiceLabel(), getTrackingId(context)); long start = System.currentTimeMillis(); try { execute(monitor, context); YSLOG.info("I04001", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), getJobLabel(), getServiceLabel(), getTrackingId(context)); } catch (IOException e) { YSLOG.error(e, "E04001", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), getJobLabel(), getServiceLabel(), getTrackingId(context)); throw e; } catch (InterruptedException e) { YSLOG.warn(e, "W04001", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), getJobLabel(), getServiceLabel(), getTrackingId(context)); throw e; } finally { YSLOG.info("I04999", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), getJobLabel(), getServiceLabel(), getTrackingId(context), end - start);
Engine( JobExecutor executor, PhaseMonitor monitor, ExecutionContext context, ErrorHandler handler, List<? extends Job> waiting) { assert executor != null; assert monitor != null; assert context != null; assert handler != null; assert waiting != null; this.executor = executor; this.monitor = monitor; this.context = context; this.handler = handler; this.waiting = new LinkedList<>(waiting); this.executing = new HashMap<>(); this.doneQueue = new LinkedBlockingQueue<>(); this.blockers = new TreeSet<>(); for (Job job : waiting) { blockers.add(job.getId()); } this.sawError = false; }
private Map<String, String> buildHadoopProperties( ExecutionContext context, HadoopScript script) throws InterruptedException, IOException { assert context != null; assert script != null; Map<String, String> props = new TreeMap<>(); props.putAll(getProperties(context, script)); props.putAll(script.getHadoopProperties()); props.put(HadoopScriptUtil.PROP_TRACKING_ID, Job.computeTrackingId(context, script)); return props; }
private boolean submitAllWaiting() throws IOException, InterruptedException { boolean sawSubmit = false; for (Iterator<Job> iter = waiting.iterator(); iter.hasNext();) { Job next = iter.next(); LOG.debug("Attemps to submit job: {}", next.getId()); if (isBlocked(next)) { LOG.debug("Job is currently blocked: {}", next.getId()); continue; } iter.remove(); if (submit(next)) { sawSubmit = true; } } return sawSubmit; }
private JobScript convert(ExecutionContext context, HadoopScript script) throws InterruptedException, IOException { assert context != null; assert script != null; JobScript result = new JobScript(); result.setBatchId(context.getBatchId()); result.setFlowId(context.getFlowId()); result.setExecutionId(context.getExecutionId()); result.setPhase(context.getPhase()); result.setArguments(new HashMap<>(context.getArguments())); result.setStageId(script.getId()); result.setMainClassName(script.getClassName()); Map<String, String> props = new HashMap<>(); props.putAll(getProperties(context, script)); props.putAll(script.getHadoopProperties()); props.put(HadoopScriptUtil.PROP_TRACKING_ID, Job.computeTrackingId(context, script)); result.setProperties(props); Map<String, String> env = new HashMap<>(); // NOTE: Handler has only dummy environment variables // env.putAll(getEnvironmentVariables(context, script)); env.putAll(context.getEnvironmentVariables()); env.putAll(script.getEnvironmentVariables()); result.setEnvironmentVariables(env); return result; }
private boolean submit(Job job) throws InterruptedException, IOException { assert job != null; monitor.checkCancelled(); try { Executing execution = executor.submit( monitor.createJobMonitor(job.getId(), 1), context, job, doneQueue); executing.put(execution.getJob().getId(), execution); return true; } catch (IOException e) { sawError = true; handleException(job, e); return false; } }
private void handleDone(Executing done) throws InterruptedException, IOException { assert done != null; assert done.isDone(); Executing removed = executing.remove(done.getJob().getId()); assert removed != null; try { done.get(); done(done.getJob()); monitor.reportJobStatus(done.getJob().getId(), JobStatus.SUCCESS, null); } catch (CancellationException e) { sawError = true; monitor.reportJobStatus(done.getJob().getId(), JobStatus.CANCELLED, e); } catch (ExecutionException e) { sawError = true; Throwable cause = e.getCause(); if (cause instanceof InterruptedException) { monitor.reportJobStatus(done.getJob().getId(), JobStatus.CANCELLED, cause); cancelExecution(); throw (InterruptedException) cause; } else if (cause instanceof IOException) { monitor.reportJobStatus(done.getJob().getId(), JobStatus.FAILED, cause); handleException(done.getJob(), (IOException) cause); } else if (cause instanceof Error) { monitor.reportJobStatus(done.getJob().getId(), JobStatus.FAILED, cause); cancelExecution(); throw (Error) cause; } else if (cause instanceof RuntimeException) { monitor.reportJobStatus(done.getJob().getId(), JobStatus.FAILED, cause); cancelExecution(); throw (RuntimeException) cause;