public static JobExecutionUpdatable createFromJobSpec(JobSpec jobSpec) { return new JobExecutionUpdatable(jobSpec.getUri(), jobSpec.getVersion(), System.currentTimeMillis(), JobLauncherUtils.newJobId(JobState.getJobNameFromProps(jobSpec.getConfigAsProperties()))); } }
/** * Create a {@link Trigger} from the given {@link JobSpec} */ private Trigger createTrigger(JobKey jobKey, JobSpec jobSpec) { // Build a trigger for the job with the given cron-style schedule return TriggerBuilder.newTrigger() .withIdentity("Cron for " + jobSpec.getUri()) .forJob(jobKey) .withSchedule(CronScheduleBuilder.cronSchedule( jobSpec.getConfig().getString(ConfigurationKeys.JOB_SCHEDULE_KEY))) .build(); }
/** * Resolve the job spec using classpath templates as well as any templates available in the input {@link JobCatalog}. */ public ResolvedJobSpec(JobSpec other, JobCatalog catalog) throws SpecNotFoundException, JobTemplate.TemplateException { super(other.getUri(), other.getVersion(), other.getDescription(), resolveConfig(other, catalog), ConfigUtils.configToProperties(resolveConfig(other, catalog)), other.getTemplateURI()); this.originalJobSpec = other; }
public String toLongString() { return getUri().toString() + "/" + getVersion() + "[" + getDescription() + "]"; }
public String toShortString() { return getUri().toString() + "/" + getVersion(); }
JobSpec.Builder jobSpecBuilder = JobSpec.builder(flowSpec.getUri()) .withConfig(flowSpec.getConfig()) .withDescription(flowSpec.getDescription()) try { jobSpec = new ResolvedJobSpec(jobSpecBuilder.build(), templateCatalog.get()); log.info("Resolved JobSpec properties are: " + jobSpec.getConfigAsProperties()); } catch (SpecNotFoundException | JobTemplate.TemplateException e) { throw new RuntimeException("Could not resolve template in JobSpec from TemplateCatalog", e); log.info("Unresolved JobSpec properties are: " + jobSpec.getConfigAsProperties()); jobSpec.setConfig(jobSpec.getConfig().withoutPath(ConfigurationKeys.JOB_SCHEDULE_KEY)); jobSpec.setConfig(jobSpec.getConfig() .withValue(ConfigurationKeys.JOB_NAME_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_NAME_KEY))); jobSpec.setConfig(jobSpec.getConfig() .withValue(ConfigurationKeys.JOB_GROUP_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_GROUP_KEY))); jobSpec.setConfig(jobSpec.getConfig().withValue(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, ConfigValueFactory.fromAnyRef(flowExecutionId))); jobSpec.setConfigAsProperties(ConfigUtils.configToProperties(jobSpec.getConfig())); for (Map.Entry<String, String> capability : capabilities.entrySet()) { log.info(String.format("Evaluating current JobSpec: %s against TopologySpec: %s with " + "capability of source: %s and destination: %s ", jobSpec.getUri(), topologySpec.getUri(), capability.getKey(), capability.getValue()));
private void fetchJobSpecs() throws ExecutionException, InterruptedException { List<Pair<SpecExecutorInstance.Verb, Spec>> changesSpecs = (List<Pair<SpecExecutorInstance.Verb, Spec>>) this.specExecutorInstanceConsumer.changedSpecs().get(); // propagate thread interruption so that caller will exit from loop if (Thread.interrupted()) { throw new InterruptedException(); } for (Pair<SpecExecutorInstance.Verb, Spec> entry : changesSpecs) { SpecExecutorInstance.Verb verb = entry.getKey(); if (verb.equals(SpecExecutorInstance.Verb.ADD)) { // Handle addition JobSpec jobSpec = (JobSpec) entry.getValue(); postNewJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); } else if (verb.equals(SpecExecutorInstanceConsumer.Verb.UPDATE)) { // Handle update JobSpec jobSpec = (JobSpec) entry.getValue(); postUpdateJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); } else if (verb.equals(SpecExecutorInstanceConsumer.Verb.DELETE)) { // Handle delete Spec anonymousSpec = (Spec) entry.getValue(); postDeleteJobConfigArrival(anonymousSpec.getUri().toString(), new Properties()); } } }
@Override protected JobSpec doPut(JobSpec jobSpec) { return this.jobSpecs.put(jobSpec.getUri(), jobSpec); }
mapBuilder.put(ImmutableFSJobCatalog.DESCRIPTION_KEY_IN_JOBSPEC, jobSpec.getDescription()) .put(ImmutableFSJobCatalog.VERSION_KEY_IN_JOBSPEC, jobSpec.getVersion()); if (jobSpec.getTemplateURI().isPresent()) { mapBuilder.put(ConfigurationKeys.JOB_TEMPLATE_PATH, jobSpec.getTemplateURI().get().toString()); String renderedConfig = ConfigFactory.parseMap(injectedKeys).withFallback(jobSpec.getConfig()) .root().render(ConfigRenderOptions.defaults()); try (DataOutputStream os = fs.create(shadowFilePath);
/** {@inheritDoc} */ @Override protected JobSpecSchedule doScheduleJob(JobSpec jobSpec, Runnable jobRunnable) { // Build a data map that gets passed to the job JobDataMap jobDataMap = new JobDataMap(); jobDataMap.put(JOB_SPEC_KEY, jobSpec); jobDataMap.put(JOB_RUNNABLE_KEY, jobRunnable); // Build a Quartz job JobDetail job = JobBuilder.newJob(QuartzJob.class) .withIdentity(jobSpec.getUri().toString()) .withDescription(Strings.nullToEmpty(jobSpec.getDescription())) .usingJobData(jobDataMap) .build(); Trigger jobTrigger = createTrigger(job.getKey(), jobSpec); QuartzJobSchedule jobSchedule = new QuartzJobSchedule(jobSpec, jobRunnable, jobTrigger); try { _scheduler.getScheduler().scheduleJob(job, jobTrigger); getLog().info(String.format("Scheduled job %s next two fire times: %s , %s.", jobSpec, jobTrigger.getNextFireTime(), jobTrigger.getFireTimeAfter(jobTrigger.getNextFireTime()))); } catch (SchedulerException e) { throw new RuntimeException("Scheduling failed for " + jobSpec + ":" + e, e); } return jobSchedule; }
@Override public void onDeleteJob(URI deletedJobURI, String deletedJobVersion) { super.onDeleteJob(deletedJobURI, deletedJobVersion); try { JobSpec.Builder jobSpecBuilder = JobSpec.builder(deletedJobURI); Properties props = new Properties(); jobSpecBuilder.withVersion(deletedJobVersion).withConfigAsProperties(props); _jobSpecQueue.put(new ImmutablePair<Verb, Spec>(Verb.DELETE, jobSpecBuilder.build())); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } }
/** {@inheritDoc} */ @Override public JobSpecSchedule scheduleJob(JobSpec jobSpec, Runnable jobRunnable) { _log.info("Scheduling JobSpec " + jobSpec); final URI jobSpecURI = jobSpec.getUri(); JobSpecSchedule newSchedule = null; Runnable runnableWithTriggerCallback = new TriggerRunnable(jobSpec, jobRunnable); synchronized (this) { JobSpecSchedule existingSchedule = _schedules.get(jobSpecURI); if (null != existingSchedule) { if (existingSchedule.getJobSpec().equals(jobSpec)) { _log.warn("Ignoring already scheduled job: " + jobSpec); return existingSchedule; } // a new job spec -- unschedule first so that we schedule the new version unscheduleJob(jobSpecURI); } newSchedule = doScheduleJob(jobSpec, runnableWithTriggerCallback); _schedules.put(jobSpecURI, newSchedule); } _callbacksDispatcher.onJobScheduled(newSchedule); return newSchedule; }
private static JobLauncher createLauncher(Configurable _sysConfig, JobSpec _jobSpec, Logger _log, Optional<String> jobLauncherType, SharedResourcesBroker<GobblinScopeTypes> instanceBroker) { if (jobLauncherType.isPresent()) { return JobLauncherFactory.newJobLauncher(_sysConfig.getConfigAsProperties(), _jobSpec.getConfigAsProperties(), jobLauncherType.get(), instanceBroker); } else { _log.info("Creating auto jobLauncher for " + _jobSpec); try { return JobLauncherFactory.newJobLauncher(_sysConfig.getConfigAsProperties(), _jobSpec.getConfigAsProperties(), instanceBroker); } catch (Exception e) { throw new RuntimeException("JobLauncher creation failed: " + e, e); } } }
private static Config resolveConfig(JobSpec jobSpec, JobCatalog catalog) throws SpecNotFoundException, JobTemplate.TemplateException { Optional<URI> templateURIOpt = jobSpec.getTemplateURI(); if (templateURIOpt.isPresent()) { JobCatalogWithTemplates catalogWithTemplates = new PackagedTemplatesJobCatalogDecorator(catalog); JobTemplate template = catalogWithTemplates.getTemplate(templateURIOpt.get()); return template.getResolvedConfig(jobSpec.getConfig()).resolve(); } else { return jobSpec.getConfig().resolve(); } }
ConfigUtils.getBoolean(state.getJobSpec().getConfig(), ConfigurationKeys.ALERT_EMAIL_ENABLED_KEY, false); boolean notificationEmailEnabled = ConfigUtils.getBoolean(state.getJobSpec().getConfig(), ConfigurationKeys.NOTIFICATION_EMAIL_ENABLED_KEY, false); LOGGER.info("Sending job failure email for job: {}", state.getJobSpec().toShortString()); EmailUtils.sendJobFailureAlertEmail(state.getJobSpec().toShortString(), getEmailBody(state, previousStatus, newStatus), 1, ConfigUtils.configToState(state.getJobSpec().getConfig())); } catch (EmailException ee) { LOGGER.error("Failed to send job failure alert email for job " + state.getJobSpec().toShortString(), ee); LOGGER.info("Sending job completion email for job: {}", state.getJobSpec().toShortString()); EmailUtils.sendJobCompletionEmail(state.getJobSpec().toShortString(), getEmailBody(state, previousStatus, newStatus), newStatus.toString(), ConfigUtils.configToState(state.getJobSpec().getConfig())); } catch (EmailException ee) { LOGGER.error("Failed to send job completion notification email for job " + state.getJobSpec().toShortString(), ee);
/** * Allow user to programmatically delete a new JobSpec. * This method is designed to be reentrant. * @param jobURI The relative Path that specified by user, need to make it into complete path. */ @Override public synchronized void remove(URI jobURI) { Preconditions.checkState(state() == State.RUNNING, String.format("%s is not running.", this.getClass().getName())); try { JobSpec jobSpec = getJobSpec(jobURI); Path jobSpecPath = getPathForURI(this.jobConfDirPath, jobURI); if (fs.exists(jobSpecPath)) { fs.delete(jobSpecPath, false); } else { LOGGER.warn("No file with URI:" + jobSpecPath + " is found. Deletion failed."); } this.listeners.onDeleteJob(jobURI, jobSpec.getVersion()); } catch (IOException e) { throw new RuntimeException("When removing a JobConf. file, issues unexpected happen:" + e.getMessage()); } catch (SpecNotFoundException e) { LOGGER.warn("No file with URI:" + jobURI + " is found. Deletion failed."); } } }
@Override public boolean apply(JobSpec input) { Preconditions.checkNotNull(input); boolean res = true; if (this.uriPredicate.isPresent()) { res &= this.uriPredicate.get().apply(input.getUri()); } if (res && this.versionPredicate.isPresent()) { res &= this.versionPredicate.get().apply(input.getVersion()); } return res; }
/*** * TODO: Change cluster code to handle Spec. Right now all job properties are needed to be in config and template is not honored * TODO: Materialized JobSpec and make use of ResolvedJobSpec * @throws ExecutionException * @throws InterruptedException */ private void fetchJobSpecs() throws ExecutionException, InterruptedException { List<Pair<SpecExecutorInstance.Verb, Spec>> changesSpecs = (List<Pair<SpecExecutorInstance.Verb, Spec>>) this.specExecutorInstanceConsumer.changedSpecs().get(); for (Pair<SpecExecutorInstance.Verb, Spec> entry : changesSpecs) { SpecExecutorInstance.Verb verb = entry.getKey(); if (verb.equals(SpecExecutorInstance.Verb.ADD)) { // Handle addition JobSpec jobSpec = (JobSpec) entry.getValue(); postNewJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); jobSpecs.put(entry.getValue().getUri(), (JobSpec) entry.getValue()); } else if (verb.equals(SpecExecutorInstanceConsumer.Verb.UPDATE)) { // Handle update JobSpec jobSpec = (JobSpec) entry.getValue(); postUpdateJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties()); jobSpecs.put(entry.getValue().getUri(), (JobSpec) entry.getValue()); } else if (verb.equals(SpecExecutorInstanceConsumer.Verb.DELETE)) { // Handle delete Spec anonymousSpec = (Spec) entry.getValue(); postDeleteJobConfigArrival(anonymousSpec.getUri().toString(), new Properties()); jobSpecs.remove(entry.getValue().getUri()); } } }
private Map<URI, JobSpec> parseJobs(Collection<JobSpec> jobSpecs) { ImmutableMap.Builder<URI, JobSpec> mapBuilder = ImmutableMap.builder(); for (JobSpec jobSpec : jobSpecs) { mapBuilder.put(jobSpec.getUri(), jobSpec); } return mapBuilder.build(); }