@Override protected JobSpec doPut(JobSpec jobSpec) { return this.jobSpecs.put(jobSpec.getUri(), jobSpec); }
private Map<URI, JobSpec> parseJobs(Collection<JobSpec> jobSpecs) { ImmutableMap.Builder<URI, JobSpec> mapBuilder = ImmutableMap.builder(); for (JobSpec jobSpec : jobSpecs) { mapBuilder.put(jobSpec.getUri(), jobSpec); } return mapBuilder.build(); }
@VisibleForTesting static String getJobName(JobSpec jobSpec) { return jobSpec.getUri().toString().replaceAll("[/.:]", "_"); }
@Override public boolean apply(JobSpec input) { Preconditions.checkNotNull(input); boolean res = true; if (this.uriPredicate.isPresent()) { res &= this.uriPredicate.get().apply(input.getUri()); } if (res && this.versionPredicate.isPresent()) { res &= this.versionPredicate.get().apply(input.getVersion()); } return res; }
public String toShortString() { return getUri().toString() + "/" + getVersion(); }
/** * Create a {@link Trigger} from the given {@link JobSpec} */ private Trigger createTrigger(JobKey jobKey, JobSpec jobSpec) { // Build a trigger for the job with the given cron-style schedule return TriggerBuilder.newTrigger() .withIdentity("Cron for " + jobSpec.getUri()) .forJob(jobKey) .withSchedule(CronScheduleBuilder.cronSchedule( jobSpec.getConfig().getString(ConfigurationKeys.JOB_SCHEDULE_KEY))) .build(); }
/** {@inheritDoc} */ @Override public JobSpecSchedule scheduleOnce(JobSpec jobSpec, Runnable jobRunnable) { _log.info("Scheduling once JobSpec " + jobSpec); Runnable runOnceRunnable = new RunOnceRunnable(jobSpec.getUri(), jobRunnable); return scheduleJob(jobSpec, runOnceRunnable); }
@Override public void remove(URI uri) { Preconditions.checkState(state() == State.RUNNING, String.format("%s is not running.", this.getClass().getName())); Preconditions.checkNotNull(uri); JobSpec jobSpec = doRemove(uri); if (null != jobSpec) { this.listeners.onDeleteJob(jobSpec.getUri(), jobSpec.getVersion()); } }
public String toLongString() { return getUri().toString() + "/" + getVersion() + "[" + getDescription() + "]"; }
/** * Allow user to programmatically add a new JobSpec. * The method will materialized the jobSpec into real file. * * @param jobSpec The target JobSpec Object to be materialized. * Noted that the URI return by getUri is a relative path. */ @Override public synchronized void put(JobSpec jobSpec) { Preconditions.checkState(state() == State.RUNNING, String.format("%s is not running.", this.getClass().getName())); Preconditions.checkNotNull(jobSpec); try { Path jobSpecPath = getPathForURI(this.jobConfDirPath, jobSpec.getUri()); materializedJobSpec(jobSpecPath, jobSpec, this.fs); } catch (IOException e) { throw new RuntimeException("When persisting a new JobSpec, unexpected issues happen:" + e.getMessage()); } catch (JobSpecNotFoundException e) { throw new RuntimeException("When replacing a existed JobSpec, unexpected issue happen:" + e.getMessage()); } }
private void submitTrackingEvent(JobSpec job, String operType) { submitTrackingEvent(job.getUri(), job.getVersion(), operType); }
/** * Allow user to programmatically add a new JobSpec. * The method will materialized the jobSpec into real file. * * @param jobSpec The target JobSpec Object to be materialized. * Noted that the URI return by getUri is a relative path. */ @Override public synchronized void put(JobSpec jobSpec) { Preconditions.checkState(state() == State.RUNNING, String.format("%s is not running.", this.getClass().getName())); Preconditions.checkNotNull(jobSpec); try { Path jobSpecPath = getPathForURI(this.jobConfDirPath, jobSpec.getUri()); boolean isUpdate = fs.exists(jobSpecPath); materializedJobSpec(jobSpecPath, jobSpec, this.fs); if (isUpdate) { this.listeners.onUpdateJob(jobSpec); } else { this.listeners.onAddJob(jobSpec); } } catch (IOException e) { throw new RuntimeException("When persisting a new JobSpec, unexpected issues happen:" + e.getMessage()); } catch (JobSpecNotFoundException e) { throw new RuntimeException("When replacing a existed JobSpec, unexpected issue happen:" + e.getMessage()); } }
private void fetchJobSpecs() throws ExecutionException, InterruptedException { List<Pair<SpecExecutorInstance.Verb, Spec>> changesSpecs = (List<Pair<SpecExecutorInstance.Verb, Spec>>) this.specExecutorInstanceConsumer.changedSpecs().get(); // propagate thread interruption so that caller will exit from loop if (Thread.interrupted()) { throw new InterruptedException(); } for (Pair<SpecExecutorInstance.Verb, Spec> entry : changesSpecs) { SpecExecutorInstance.Verb verb = entry.getKey(); if (verb.equals(SpecExecutorInstance.Verb.ADD)) { // Handle addition JobSpec jobSpec = (JobSpec) entry.getValue(); postNewJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); } else if (verb.equals(SpecExecutorInstanceConsumer.Verb.UPDATE)) { // Handle update JobSpec jobSpec = (JobSpec) entry.getValue(); postUpdateJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); } else if (verb.equals(SpecExecutorInstanceConsumer.Verb.DELETE)) { // Handle delete Spec anonymousSpec = (Spec) entry.getValue(); postDeleteJobConfigArrival(anonymousSpec.getUri().toString(), new Properties()); } } }
@Override public JobExecutionDriver launchJob(JobSpec jobSpec) { Preconditions.checkNotNull(jobSpec); if (!(jobSpec instanceof ResolvedJobSpec)) { try { jobSpec = new ResolvedJobSpec(jobSpec); } catch (JobTemplate.TemplateException | SpecNotFoundException exc) { throw new RuntimeException("Can't launch job " + jobSpec.getUri(), exc); } } return JobLauncherExecutionDriver.create(getSysConfig(), jobSpec, _jobLauncherType, Optional.of(getLog(jobSpec)), isInstrumentationEnabled(), getMetrics(), getInstanceBroker()); }
/** {@inheritDoc} */ @Override public JobSpecSchedule scheduleJob(JobSpec jobSpec, Runnable jobRunnable) { _log.info("Scheduling JobSpec " + jobSpec); final URI jobSpecURI = jobSpec.getUri(); JobSpecSchedule newSchedule = null; Runnable runnableWithTriggerCallback = new TriggerRunnable(jobSpec, jobRunnable); synchronized (this) { JobSpecSchedule existingSchedule = _schedules.get(jobSpecURI); if (null != existingSchedule) { if (existingSchedule.getJobSpec().equals(jobSpec)) { _log.warn("Ignoring already scheduled job: " + jobSpec); return existingSchedule; } // a new job spec -- unschedule first so that we schedule the new version unscheduleJob(jobSpecURI); } newSchedule = doScheduleJob(jobSpec, runnableWithTriggerCallback); _schedules.put(jobSpecURI, newSchedule); } _callbacksDispatcher.onJobScheduled(newSchedule); return newSchedule; }
public static JobExecutionUpdatable createFromJobSpec(JobSpec jobSpec) { return new JobExecutionUpdatable(jobSpec.getUri(), jobSpec.getVersion(), System.currentTimeMillis(), JobLauncherUtils.newJobId(JobState.getJobNameFromProps(jobSpec.getConfigAsProperties()))); } }
/** {@inheritDoc} */ @Override protected JobSpecSchedule doScheduleJob(JobSpec jobSpec, Runnable jobRunnable) { // Build a data map that gets passed to the job JobDataMap jobDataMap = new JobDataMap(); jobDataMap.put(JOB_SPEC_KEY, jobSpec); jobDataMap.put(JOB_RUNNABLE_KEY, jobRunnable); // Build a Quartz job JobDetail job = JobBuilder.newJob(QuartzJob.class) .withIdentity(jobSpec.getUri().toString()) .withDescription(Strings.nullToEmpty(jobSpec.getDescription())) .usingJobData(jobDataMap) .build(); Trigger jobTrigger = createTrigger(job.getKey(), jobSpec); QuartzJobSchedule jobSchedule = new QuartzJobSchedule(jobSpec, jobRunnable, jobTrigger); try { _scheduler.getScheduler().scheduleJob(job, jobTrigger); getLog().info(String.format("Scheduled job %s next two fire times: %s , %s.", jobSpec, jobTrigger.getNextFireTime(), jobTrigger.getFireTimeAfter(jobTrigger.getNextFireTime()))); } catch (SchedulerException e) { throw new RuntimeException("Scheduling failed for " + jobSpec + ":" + e, e); } return jobSchedule; }
private AvroJobSpec convertToAvroJobSpec(Spec spec, Verb verb) { if (spec instanceof JobSpec) { JobSpec jobSpec = (JobSpec) spec; AvroJobSpec.Builder avroJobSpecBuilder = AvroJobSpec.newBuilder(); avroJobSpecBuilder.setUri(jobSpec.getUri().toString()).setVersion(jobSpec.getVersion()) .setDescription(jobSpec.getDescription()).setProperties(Maps.fromProperties(jobSpec.getConfigAsProperties())) .setMetadata(ImmutableMap.of(VERB_KEY, verb.name())); if (jobSpec.getTemplateURI().isPresent()) { avroJobSpecBuilder.setTemplateUri(jobSpec.getTemplateURI().get().toString()); } return avroJobSpecBuilder.build(); } else { throw new RuntimeException("Unsupported spec type " + spec.getClass()); } } }
/** * Resolve the job spec using classpath templates as well as any templates available in the input {@link JobCatalog}. */ public ResolvedJobSpec(JobSpec other, JobCatalog catalog) throws SpecNotFoundException, JobTemplate.TemplateException { super(other.getUri(), other.getVersion(), other.getDescription(), resolveConfig(other, catalog), ConfigUtils.configToProperties(resolveConfig(other, catalog)), other.getTemplateURI()); this.originalJobSpec = other; }
return Lists.newArrayList(Either.<JobSpec, URI>left(jobSpec)); } else { return Lists.newArrayList(Either.<JobSpec, URI>right(jobSpec.getUri()));