@Override public void launchJob(@Nullable JobListener jobListener) throws JobException { this.jobLauncher.launchJob(jobListener); }
@Override public void cancelJob(@Nullable JobListener jobListener) throws JobException { this.jobLauncher.cancelJob(jobListener); }
closer.register(jobLauncher).launchJob(jobListener); boolean runOnce = Boolean.valueOf(jobProps.getProperty(ConfigurationKeys.JOB_RUN_ONCE_KEY, "false")); boolean isEarlyStopped = jobLauncher.isEarlyStopped(); if (!isEarlyStopped && runOnce && this.scheduledJobs.containsKey(jobName)) { this.scheduler.getScheduler().deleteJob(this.scheduledJobs.remove(jobName));
@Override public JobExecutionResult call() throws JobException, InterruptedException, TimeoutException { jobLauncher.launchJob(bridge); jobState.awaitForDone(Long.MAX_VALUE); return JobExecutionResult.createFromState(jobState); } }
@Override public boolean cancel(boolean mayInterruptIfRunning) { if (!cancelRequested) { return false; } boolean result = true; try { jobLauncher.cancelJob(jobListener); } catch (JobException e) { LOG.error("Failed to cancel job " + jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY), e); result = false; } if (mayInterruptIfRunning) { result &= future.cancel(true); } return result; }
closer.register(jobLauncher).launchJob(jobListener); boolean runOnce = Boolean.valueOf(jobProps.getProperty(ConfigurationKeys.JOB_RUN_ONCE_KEY, "false")); boolean isEarlyStopped = jobLauncher.isEarlyStopped(); if (!isEarlyStopped && runOnce && this.scheduledJobs.containsKey(jobName)) { this.scheduler.getScheduler().deleteJob(this.scheduledJobs.remove(jobName));
@Test public void testTarGzCopy() throws Exception { Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(gobblinProps, jobProps)); jobLauncher.launchJob(null); String file1Path = gobblinProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR) + "/LogData/sub1/sub2/text1.txt"; String file2Path = gobblinProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR) + "/LogData/sub1/sub2/text2.txt"; FileSystem fs = FileSystem.getLocal(new Configuration()); Assert.assertEquals(IOUtils.toString(closer.register(fs.open(new Path(file1Path)))), "text1"); Assert.assertEquals(IOUtils.toString(closer.register(fs.open(new Path(file2Path)))), "text2"); } finally { closer.close(); } }
@Override public void run() { try { Thread.sleep(500); jobLauncher.cancelJob(null); isCancelled.set(true); } catch (Exception je) { // Ignored } } });
@Test(dependsOnMethods = "testLaunchSecondJob") public void testLaunchThirdJob() throws Exception { try (JobLauncher launcher = new LocalJobLauncher(this.jobConfig)) { launcher.launchJob(null); } verifyJobState(3); }
@Override public boolean cancel(boolean mayInterruptIfRunning) { // FIXME there is a race condition here as the job may complete successfully before we // call cancelJob() below. There isn't an easy way to fix that right now. RunningState runState = fetchRunningState(); if (runState.isCancelled()) { return true; } else if (runState.isDone()) { return false; } try { // No special processing of callbacks necessary getLegacyLauncher().cancelJob(new AbstractJobListener(){}); } catch (JobException e) { throw new RuntimeException("Unable to cancel job " + _jobSpec + ": " + e, e); } return super.cancel(mayInterruptIfRunning); }
@Test public void testLaunchFirstJob() throws Exception { try (JobLauncher launcher = new LocalJobLauncher(this.jobConfig)) { launcher.launchJob(null); } verifyJobState(1); }
@Override public void cancelJob(@Nullable JobListener jobListener) throws JobException { this.jobLauncher.cancelJob(jobListener); }
@Test(dependsOnMethods = "testLaunchFirstJob") public void testLaunchSecondJob() throws Exception { try (JobLauncher launcher = new LocalJobLauncher(this.jobConfig)) { launcher.launchJob(null); } verifyJobState(2); }
@Override public boolean cancel(boolean mayInterruptIfRunning) { if (!cancelRequested) { return false; } boolean result = true; try { jobLauncher.cancelJob(jobListener); } catch (JobException e) { LOG.error("Failed to cancel job " + jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY), e); result = false; } if (mayInterruptIfRunning) { result &= future.cancel(true); } return result; }
public void runTestWithCancellation(final Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); Closer closer = Closer.create(); try { final JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); final AtomicBoolean isCancelled = new AtomicBoolean(false); // This thread will cancel the job after some time Thread thread = new Thread(new Runnable() { @Override public void run() { try { Thread.sleep(500); jobLauncher.cancelJob(null); isCancelled.set(true); } catch (Exception je) { // Ignored } } }); thread.start(); jobLauncher.launchJob(null); Assert.assertTrue(isCancelled.get()); } finally { closer.close(); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); Assert.assertTrue(datasetStateList.isEmpty()); }
@Override public boolean cancel(boolean mayInterruptIfRunning) { // FIXME there is a race condition here as the job may complete successfully before we // call cancelJob() below. There isn't an easy way to fix that right now. RunningState runState = fetchRunningState(); if (runState.isCancelled()) { return true; } else if (runState.isDone()) { return false; } try { // No special processing of callbacks necessary getLegacyLauncher().cancelJob(new AbstractJobListener(){}); } catch (JobException e) { throw new RuntimeException("Unable to cancel job " + _jobSpec + ": " + e, e); } return super.cancel(mayInterruptIfRunning); }
totalSuccessful = 5; // number of records in partition 1 JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(gobblinProps, jobProps)); jobLauncher.launchJob(null); jobProps.setProperty("writer.kafka.producerConfig.flaky.errorEvery", "" + errorEvery); JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(gobblinProps, jobProps)); jobLauncher.launchJob(null); totalSuccessful = totalRecords / errorEvery + totalRecords%errorEvery; } catch (Exception e) {
public void runTestWithFork(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); try (JobLauncher jobLauncher = JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)) { jobLauncher.launchJob(null); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 4); Assert.assertEquals(datasetState.getJobFailures(), 0); FileSystem lfs = FileSystem.getLocal(new Configuration()); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Path path = new Path(this.launcherProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR), new Path(taskState.getExtract().getOutputFilePath(), "fork_0")); Assert.assertTrue(lfs.exists(path)); Assert.assertEquals(lfs.listStatus(path).length, 2); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN + ".0"), TestExtractor.TOTAL_RECORDS); path = new Path(this.launcherProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR), new Path(taskState.getExtract().getOutputFilePath(), "fork_1")); Assert.assertTrue(lfs.exists(path)); Assert.assertEquals(lfs.listStatus(path).length, 2); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN + ".1"), TestExtractor.TOTAL_RECORDS); } }
public void runTestWithCommitSuccessfulTasksPolicy(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); jobProps.setProperty(ConfigurationKeys.PUBLISH_DATA_AT_JOB_LEVEL, Boolean.FALSE.toString()); jobProps.setProperty(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "successful"); jobProps.setProperty(ConfigurationKeys.SOURCE_CLASS_KEY, TestSourceWithFaultyExtractor.class.getName()); jobProps.setProperty(ConfigurationKeys.MAX_TASK_RETRIES_KEY, "0"); Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally { closer.close(); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); JobState jobState = datasetStateList.get(0); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getCompletedTasks(), 4); for (TaskState taskState : jobState.getTaskStates()) { if (taskState.getTaskId().endsWith("0")) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.FAILED); } else { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS); } } }
public void runTestWithPullLimit(Properties jobProps, long limit) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally { closer.close(); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 4); Assert.assertEquals(datasetState.getJobFailures(), 0); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.EXTRACTOR_ROWS_EXTRACTED), limit); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_ROWS_WRITTEN), limit); } }