/** * Get the number of records written by all the writers * @return Sum of the writer records written count across all tasks */ public static long getProcessedCount(List<TaskState> taskStates) { long value = 0; for (TaskState taskState : taskStates) { value += taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN, 0); } return value; }
/** * Get the interval for status reporting. * * @return interval for status reporting */ public long getStatusReportingInterval() { return this.taskState.getPropAsLong(ConfigurationKeys.TASK_STATUS_REPORT_INTERVAL_IN_MS_KEY, ConfigurationKeys.DEFAULT_TASK_STATUS_REPORT_INTERVAL_IN_MS); }
public SleepingTask(TaskContext taskContext) { super(taskContext); sleepTime = taskContext.getTaskState().getPropAsLong("data.publisher.sleep.time.in.seconds", 10L); }
private void onStart(long startTime) { Long queueTime = queuedTasks.remove(this.underlyingTask.getTaskId()); long workUnitCreationTime = this.underlyingTask.getTaskContext().getTaskState().getPropAsLong(ConfigurationKeys.WORK_UNIT_CREATION_TIME_IN_MILLIS, 0); long timeInQueue = startTime - queueTime; long timeSinceWorkUnitCreation = startTime - workUnitCreationTime; taskCreateAndRunTimer.update(timeSinceWorkUnitCreation, TimeUnit.MILLISECONDS); LOG.debug(String.format("Task %s started. Saving queued time of %d ms to history.", underlyingTask.getTaskId(), timeInQueue)); queuedTaskTimeHistorical.putIfAbsent(System.currentTimeMillis(), timeInQueue); runningTaskCount.inc(); } }
public void runTestWithPullLimit(Properties jobProps, long limit) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally { closer.close(); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 4); Assert.assertEquals(datasetState.getJobFailures(), 0); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.EXTRACTOR_ROWS_EXTRACTED), limit); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_ROWS_WRITTEN), limit); } }
public void runTestWithFork(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); try (JobLauncher jobLauncher = JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)) { jobLauncher.launchJob(null); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 4); Assert.assertEquals(datasetState.getJobFailures(), 0); FileSystem lfs = FileSystem.getLocal(new Configuration()); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Path path = new Path(this.launcherProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR), new Path(taskState.getExtract().getOutputFilePath(), "fork_0")); Assert.assertTrue(lfs.exists(path)); Assert.assertEquals(lfs.listStatus(path).length, 2); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN + ".0"), TestExtractor.TOTAL_RECORDS); path = new Path(this.launcherProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR), new Path(taskState.getExtract().getOutputFilePath(), "fork_1")); Assert.assertTrue(lfs.exists(path)); Assert.assertEquals(lfs.listStatus(path).length, 2); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN + ".1"), TestExtractor.TOTAL_RECORDS); } }
if (errRecords > this.taskState.getPropAsLong(TaskConfigurationKeys.TASK_SKIP_ERROR_RECORDS, TaskConfigurationKeys.DEFAULT_TASK_SKIP_ERROR_RECORDS)) { throw new RuntimeException(e);
public void runTestWithCommitSuccessfulTasksPolicy(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); jobProps.setProperty(ConfigurationKeys.PUBLISH_DATA_AT_JOB_LEVEL, Boolean.FALSE.toString()); jobProps.setProperty(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "successful"); jobProps.setProperty(ConfigurationKeys.SOURCE_CLASS_KEY, TestSourceWithFaultyExtractor.class.getName()); jobProps.setProperty(ConfigurationKeys.MAX_TASK_RETRIES_KEY, "0"); Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally { closer.close(); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); JobState jobState = datasetStateList.get(0); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getCompletedTasks(), 4); for (TaskState taskState : jobState.getTaskStates()) { if (taskState.getTaskId().endsWith("0")) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.FAILED); } else { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS); } } }
public AsynchronousFork(TaskContext taskContext, Object schema, int branches, int index, ExecutionModel executionModel) throws Exception { super(taskContext, schema, branches, index, executionModel); TaskState taskState = taskContext.getTaskState(); this.recordQueue = BoundedBlockingRecordQueue.newBuilder() .hasCapacity(taskState.getPropAsInt( ConfigurationKeys.FORK_RECORD_QUEUE_CAPACITY_KEY, ConfigurationKeys.DEFAULT_FORK_RECORD_QUEUE_CAPACITY)) .useTimeout(taskState.getPropAsLong( ConfigurationKeys.FORK_RECORD_QUEUE_TIMEOUT_KEY, ConfigurationKeys.DEFAULT_FORK_RECORD_QUEUE_TIMEOUT)) .useTimeoutTimeUnit(TimeUnit.valueOf(taskState.getProp( ConfigurationKeys.FORK_RECORD_QUEUE_TIMEOUT_UNIT_KEY, ConfigurationKeys.DEFAULT_FORK_RECORD_QUEUE_TIMEOUT_UNIT))) .collectStats() .build(); }
public void runTestWithMultipleDatasets(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); jobProps.setProperty(ConfigurationKeys.SOURCE_CLASS_KEY, MultiDatasetTestSource.class.getName()); Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally { closer.close(); } for (int i = 0; i < 4; i++) { List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, "Dataset" + i + "-current.jst"); DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getDatasetUrn(), "Dataset" + i); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 1); Assert.assertEquals(datasetState.getJobFailures(), 0); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getProp(ConfigurationKeys.DATASET_URN_KEY), "Dataset" + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS); } } }
} else { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS);
public void runTest(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); JobContext jobContext = null; Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); jobContext = ((AbstractJobLauncher) jobLauncher).getJobContext(); } finally { closer.close(); } Assert.assertTrue(jobContext.getJobMetricsOptional().isPresent()); String jobMetricContextTags = jobContext.getJobMetricsOptional().get().getMetricContext().getTags().toString(); Assert.assertTrue(jobMetricContextTags.contains(ClusterNameTags.CLUSTER_IDENTIFIER_TAG_NAME), ClusterNameTags.CLUSTER_IDENTIFIER_TAG_NAME + " tag missing in job metric context tags."); List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 4); Assert.assertEquals(datasetState.getJobFailures(), 0); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS); } }
protected void run() throws Exception { long maxWaitInMinute = taskState.getPropAsLong(ConfigurationKeys.FORK_MAX_WAIT_MININUTES, ConfigurationKeys.DEFAULT_FORK_MAX_WAIT_MININUTES);
/** * Get the interval for status reporting. * * @return interval for status reporting */ public long getStatusReportingInterval() { return this.taskState.getPropAsLong(ConfigurationKeys.TASK_STATUS_REPORT_INTERVAL_IN_MS_KEY, ConfigurationKeys.DEFAULT_TASK_STATUS_REPORT_INTERVAL_IN_MS); }
/** * Get the number of records written by all the writers * @return Sum of the writer records written count across all tasks */ public static long getProcessedCount(List<TaskState> taskStates) { long value = 0; for (TaskState taskState : taskStates) { value += taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN, 0); } return value; }
private void onStart(long startTime) { Long queueTime = queuedTasks.remove(this.underlyingTask.getTaskId()); long workUnitCreationTime = this.underlyingTask.getTaskContext().getTaskState().getPropAsLong(ConfigurationKeys.WORK_UNIT_CREATION_TIME_IN_MILLIS, 0); long timeInQueue = startTime - queueTime; long timeSinceWorkUnitCreation = startTime - workUnitCreationTime; taskCreateAndRunTimer.update(timeSinceWorkUnitCreation, TimeUnit.MILLISECONDS); LOG.debug(String.format("Task %s started. Saving queued time of %d ms to history.", underlyingTask.getTaskId(), timeInQueue)); queuedTaskTimeHistorical.putIfAbsent(System.currentTimeMillis(), timeInQueue); runningTaskCount.inc(); } }
if (errRecords > this.taskState.getPropAsLong(TaskConfigurationKeys.TASK_SKIP_ERROR_RECORDS, TaskConfigurationKeys.DEFAULT_TASK_SKIP_ERROR_RECORDS)) { throw new RuntimeException(e);
public AsynchronousFork(TaskContext taskContext, Object schema, int branches, int index, ExecutionModel executionModel) throws Exception { super(taskContext, schema, branches, index, executionModel); TaskState taskState = taskContext.getTaskState(); this.recordQueue = BoundedBlockingRecordQueue.newBuilder() .hasCapacity(taskState.getPropAsInt( ConfigurationKeys.FORK_RECORD_QUEUE_CAPACITY_KEY, ConfigurationKeys.DEFAULT_FORK_RECORD_QUEUE_CAPACITY)) .useTimeout(taskState.getPropAsLong( ConfigurationKeys.FORK_RECORD_QUEUE_TIMEOUT_KEY, ConfigurationKeys.DEFAULT_FORK_RECORD_QUEUE_TIMEOUT)) .useTimeoutTimeUnit(TimeUnit.valueOf(taskState.getProp( ConfigurationKeys.FORK_RECORD_QUEUE_TIMEOUT_UNIT_KEY, ConfigurationKeys.DEFAULT_FORK_RECORD_QUEUE_TIMEOUT_UNIT))) .collectStats() .build(); }
protected void run() throws Exception { long maxWaitInMinute = taskState.getPropAsLong(ConfigurationKeys.FORK_MAX_WAIT_MININUTES, ConfigurationKeys.DEFAULT_FORK_MAX_WAIT_MININUTES);