void checkForUnpublishedWUHandling(String datasetUrn, JobState.DatasetState datasetState, Class<? extends DataPublisher> dataPublisherClass, Closer closer) throws ReflectiveOperationException, IOException { if (UnpublishedHandling.class.isAssignableFrom(dataPublisherClass)) { // pass in jobstate to retrieve properties DataPublisher publisher = closer.register(DataPublisher.getInstance(dataPublisherClass, this.jobContext.getJobState())); log.info(String.format("Calling publisher to handle unpublished work units for dataset %s of job %s.", datasetUrn, this.jobContext.getJobId())); ((UnpublishedHandling) publisher).handleUnpublishedWorkUnits(datasetState.getTaskStatesAsWorkUnitStates()); } }
@Override public String toString() { return Objects.toStringHelper(JobContext.class.getSimpleName()).add("jobName", getJobName()) .add("jobId", getJobId()).add("jobState", getJobState()).toString(); } }
LOG.info("Current job state is: " + this.jobContext.getJobState().getState()); if (this.jobContext.getJobState().getState() != JobState.RunningState.COMMITTED && ( this.jobContext.getJobCommitPolicy() == JobCommitPolicy.COMMIT_SUCCESSFUL_TASKS || this.jobContext.getJobCommitPolicy() == JobCommitPolicy.COMMIT_ON_PARTIAL_SUCCESS)) { this.jobContext.finalizeJobStateBeforeCommit(); this.jobContext.commit(true); this.jobContext.close(); } catch (IOException ioe) { LOG.error("Could not close job context.", ioe);
/** * FIXME this method is provided for backwards compatibility in the LocalJobLauncher since it does * not access the task state store. This should be addressed as all task executions should be * updating the task state. */ public static GobblinMultiTaskAttempt runWorkUnits(JobContext jobContext, Iterator<WorkUnit> workUnits, TaskStateTracker taskStateTracker, TaskExecutor taskExecutor, CommitPolicy multiTaskAttemptCommitPolicy) throws IOException, InterruptedException { GobblinMultiTaskAttempt multiTaskAttempt = new GobblinMultiTaskAttempt(workUnits, jobContext.getJobId(), jobContext.getJobState(), taskStateTracker, taskExecutor, Optional.<String>absent(), Optional.<StateStore<TaskState>>absent(), jobContext.getJobBroker()); multiTaskAttempt.runAndOptionallyCommitTaskAttempt(multiTaskAttemptCommitPolicy); return multiTaskAttempt; }
@Override public void launchJob(JobListener jobListener) throws JobException { String jobId = this.jobContext.getJobId(); final JobState jobState = this.jobContext.getJobState(); boolean isWorkUnitsEmpty = false; MDC.put(ConfigurationKeys.JOB_NAME_KEY, this.jobContext.getJobName()); MDC.put(ConfigurationKeys.JOB_KEY_KEY, this.jobContext.getJobKey()); TimingEvent launchJobTimer = this.eventSubmitter.getTimingEvent(TimingEvent.LauncherTimings.FULL_JOB_EXECUTION); if (this.jobContext.getSemantics() == DeliverySemantics.EXACTLY_ONCE) { Source<?, ?> source = this.jobContext.getSource(); WorkUnitStream workUnitStream; if (source instanceof WorkUnitStreamSource) { this.jobContext.storeJobExecutionInfo(); this.jobContext.finalizeJobStateBeforeCommit(); this.jobContext.commit(); postProcessJobState(jobState); jobCommitTimer.stop(this.eventMetadataGenerator.getMetadata(this.jobContext, EventName.JOB_COMMIT)); this.jobContext.storeJobExecutionInfo(); } finally { launchJobTimer.stop(this.eventMetadataGenerator.getMetadata(this.jobContext, EventName.FULL_JOB_EXECUTION)); for (JobState.DatasetState datasetState : this.jobContext.getDatasetStatesByUrns().values()) {
@Override public void onJobCancellation(JobContext jobContext) { this.jobExecutionEventSubmitter.submitJobExecutionEvents(jobContext.getJobState()); } }
Class<? extends DataPublisher> dataPublisherClass; try (Closer closer = Closer.create()) { dataPublisherClass = JobContext.getJobDataPublisherClass(this.jobContext.getJobState()) .or((Class<? extends DataPublisher>) Class.forName(ConfigurationKeys.DEFAULT_DATA_PUBLISHER_TYPE)); if (!canCommitDataset(datasetState)) { log.warn(String .format("Not committing dataset %s of job %s with commit policy %s and state %s", this.datasetUrn, this.jobContext.getJobId(), this.jobContext.getJobCommitPolicy(), this.datasetState.getState())); checkForUnpublishedWUHandling(this.datasetUrn, this.datasetState, dataPublisherClass, closer); throw new RuntimeException(String .format("Not committing dataset %s of job %s with commit policy %s and state %s", this.datasetUrn, this.jobContext.getJobId(), this.jobContext.getJobCommitPolicy(), this.datasetState.getState())); this.jobContext.getJobId(), roe); throw new RuntimeException(roe); } finally { .getJobCommitPolicy()); if (this.shouldCommitDataInJob) { log.info(String.format("Committing dataset %s of job %s with commit policy %s and state %s", this.datasetUrn, this.jobContext.getJobId(), this.jobContext.getJobCommitPolicy(), this.datasetState.getState())); publisher = DataPublisherFactory.get(dataPublisherClass.getName(), this.jobContext.getJobState(), this.jobContext.getJobBroker()); this.jobContext.getJobId()), throwable); throw new RuntimeException(throwable); } finally {
public String getJobId() { return this.jobContext.getJobId(); }
Object[] currentJob = new Object[]{JOB_PREFIX, JOB_NAME1, timestamp2}; Path currentJobPath = new Path(JobContext.getJobDir(rootPath, JOB_NAME1), JOINER.join(currentJob)); Path oldJobPath1 = new Path(JobContext.getJobDir(rootPath, JOB_NAME1), JOINER.join(oldJob1)); Path oldJobPath2 = new Path(JobContext.getJobDir(rootPath, JOB_NAME2), JOINER.join(oldJob2)); Path stagingPath = new Path(currentJobPath, "task-staging"); Path outputPath = new Path(currentJobPath, "task-output"); when(jobContext.getStagingDirProvided()).thenReturn(false); when(jobContext.getOutputDirProvided()).thenReturn(false); when(jobContext.getJobId()).thenReturn(currentJobPath.getName().toString()); JobLauncherUtils.cleanUpOldJobData(jobState, log, jobContext.getStagingDirProvided(), jobContext.getOutputDirProvided());
@Override public void close() throws IOException { try { this.cancellationExecutor.shutdownNow(); try { this.jobContext.getSource().shutdown(this.jobContext.getJobState()); } finally { if (GobblinMetrics.isEnabled(this.jobProps)) { GobblinMetricsRegistry.getInstance().remove(this.jobContext.getJobId()); } } } finally { unlockJob(); } }
public void launchJob(@Nullable JobListener jobListener) throws JobException { this.jobListener = jobListener; boolean isLaunched = false; this.runningMap.putIfAbsent(this.jobContext.getJobName(), false); try { if (this.runningMap.replace(this.jobContext.getJobName(), false, true)) { LOGGER.info ("Job {} will be executed, add into running map.", this.jobContext.getJobId()); isLaunched = true; super.launchJob(jobListener); } else { LOGGER.warn ("Job {} will not be executed because other jobs are still running.", this.jobContext.getJobId()); } } finally { if (isLaunched) { if (this.runningMap.replace(this.jobContext.getJobName(), true, false)) { LOGGER.info ("Job {} is done, remove from running map.", this.jobContext.getJobId()); } else { throw new IllegalStateException("A launched job should have running state equal to true in the running map."); } } } }
protected void setTaskStagingAndOutputDirs() { // Add jobId to writer staging dir if (this.jobState.contains(ConfigurationKeys.WRITER_STAGING_DIR)) { String writerStagingDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.WRITER_STAGING_DIR), this.getJobName()), this.jobId).toString(); this.jobState.setProp(ConfigurationKeys.WRITER_STAGING_DIR, writerStagingDirWithJobId); } // Add jobId to writer output dir if (this.jobState.contains(ConfigurationKeys.WRITER_OUTPUT_DIR)) { String writerOutputDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR), this.getJobName()), this.jobId) .toString(); this.jobState.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, writerOutputDirWithJobId); } // Add jobId to task data root dir if (this.jobState.contains(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY)) { String taskDataRootDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY), this.getJobName()), this.jobId).toString(); this.jobState.setProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY, taskDataRootDirWithJobId); setTaskStagingDir(); setTaskOutputDir(); } else { LOG.warn("Property " + ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY + " is missing."); } }
this.jobContext = new JobContext(this.jobProps, LOG, instanceBroker); this.eventBus.register(this.jobContext); this.jobContext.getJobMetricsOptional().transform(new Function<JobMetrics, MetricContext>() { @Override public MetricContext apply(JobMetrics input) { GobblinMetrics.addCustomTagToState(this.jobContext.getJobState(), metadataTags);
@Override protected void runWorkUnits(List<WorkUnit> workUnits) throws Exception { String jobName = this.jobContext.getJobName(); JobState jobState = this.jobContext.getJobState();
if (this.jobContext.shouldCleanupStagingDataPerTask()) { if (workUnits.isSafeToMaterialize()) { Closer closer = Closer.create(); JobLauncherUtils.cleanUpOldJobData(jobState, LOG, jobContext.getStagingDirProvided(), jobContext.getOutputDirProvided());
/** * Commit the job on a per-dataset basis. */ void commit() throws IOException { commit(false); }
public void runTest(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); JobContext jobContext = null; Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); jobContext = ((AbstractJobLauncher) jobLauncher).getJobContext(); } finally { closer.close(); } Assert.assertTrue(jobContext.getJobMetricsOptional().isPresent()); String jobMetricContextTags = jobContext.getJobMetricsOptional().get().getMetricContext().getTags().toString(); Assert.assertTrue(jobMetricContextTags.contains(ClusterNameTags.CLUSTER_IDENTIFIER_TAG_NAME), ClusterNameTags.CLUSTER_IDENTIFIER_TAG_NAME + " tag missing in job metric context tags."); List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 4); Assert.assertEquals(datasetState.getJobFailures(), 0); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS); } }
@Override public void launchJob(JobListener jobListener) throws JobException { String jobId = this.jobContext.getJobId(); final JobState jobState = this.jobContext.getJobState(); MDC.put(ConfigurationKeys.JOB_NAME_KEY, this.jobContext.getJobName()); MDC.put(ConfigurationKeys.JOB_KEY_KEY, this.jobContext.getJobKey()); TimingEvent launchJobTimer = this.eventSubmitter.getTimingEvent(TimingEvent.LauncherTimings.FULL_JOB_EXECUTION); if (this.jobContext.getSemantics() == DeliverySemantics.EXACTLY_ONCE) { Source<?, ?> source = this.jobContext.getSource(); WorkUnitStream workUnitStream; if (source instanceof WorkUnitStreamSource) { this.jobContext.storeJobExecutionInfo(); this.jobContext.finalizeJobStateBeforeCommit(); this.jobContext.commit(); postProcessJobState(jobState); jobCommitTimer.stop(this.eventMetadataGenerator.getMetadata(this.jobContext, EventName.JOB_COMMIT)); this.jobContext.storeJobExecutionInfo(); } finally { launchJobTimer.stop(this.eventMetadataGenerator.getMetadata(this.jobContext, EventName.FULL_JOB_EXECUTION)); for (JobState.DatasetState datasetState : this.jobContext.getDatasetStatesByUrns().values()) { if (this.jobContext.getJobMetricsOptional().isPresent()) {
@Override public void onJobCompletion(JobContext jobContext) { this.jobExecutionEventSubmitter.submitJobExecutionEvents(jobContext.getJobState()); }
Class<? extends DataPublisher> dataPublisherClass; try (Closer closer = Closer.create()) { dataPublisherClass = JobContext.getJobDataPublisherClass(this.jobContext.getJobState()) .or((Class<? extends DataPublisher>) Class.forName(ConfigurationKeys.DEFAULT_DATA_PUBLISHER_TYPE)); if (!canCommitDataset(datasetState)) { log.warn(String .format("Not committing dataset %s of job %s with commit policy %s and state %s", this.datasetUrn, this.jobContext.getJobId(), this.jobContext.getJobCommitPolicy(), this.datasetState.getState())); checkForUnpublishedWUHandling(this.datasetUrn, this.datasetState, dataPublisherClass, closer); throw new RuntimeException(String .format("Not committing dataset %s of job %s with commit policy %s and state %s", this.datasetUrn, this.jobContext.getJobId(), this.jobContext.getJobCommitPolicy(), this.datasetState.getState())); this.jobContext.getJobId(), roe); throw new RuntimeException(roe); } finally { .getJobCommitPolicy()); if (this.shouldCommitDataInJob) { log.info(String.format("Committing dataset %s of job %s with commit policy %s and state %s", this.datasetUrn, this.jobContext.getJobId(), this.jobContext.getJobCommitPolicy(), this.datasetState.getState())); publisher = DataPublisherFactory.get(dataPublisherClass.getName(), this.jobContext.getJobState(), this.jobContext.getJobBroker()); this.jobContext.getJobId()), throwable); throw new RuntimeException(throwable); } finally {