/** * Return whether the task has completed running or not. * * @return {@code true} if the task has completed or {@code false} otherwise */ public boolean isCompleted() { WorkingState state = getWorkingState(); return state == WorkingState.SUCCESSFUL || state == WorkingState.COMMITTED || state == WorkingState.FAILED; }
@Override public WorkUnitState.WorkingState getWorkingState() { return getTaskState().getWorkingState(); }
/** * Filter the task states corresponding to the skipped work units and add it to the skippedTaskStates */ public void filterSkippedTaskStates() { List<TaskState> skippedTaskStates = new ArrayList<>(); for (TaskState taskState : this.taskStates.values()) { if (taskState.getWorkingState() == WorkUnitState.WorkingState.SKIPPED) { skippedTaskStates.add(taskState); } } for (TaskState taskState : skippedTaskStates) { removeTaskState(taskState); addSkippedTaskState(taskState); } }
/** * Submits an event for a given {@link TaskState}. It will include all metadata specified in the jobMetadata parameter. */ private void submitTaskStateEvent(TaskState taskState, Map<String, String> jobMetadata) { ImmutableMap.Builder<String, String> taskMetadataBuilder = new ImmutableMap.Builder<>(); taskMetadataBuilder.putAll(jobMetadata); taskMetadataBuilder.put(METADATA_TASK_ID, taskState.getTaskId()); taskMetadataBuilder.put(METADATA_TASK_START_TIME, Long.toString(taskState.getStartTime())); taskMetadataBuilder.put(METADATA_TASK_END_TIME, Long.toString(taskState.getEndTime())); taskMetadataBuilder.put(METADATA_TASK_WORKING_STATE, taskState.getWorkingState().toString()); taskMetadataBuilder.put(METADATA_TASK_FAILURE_CONTEXT, taskState.getTaskFailureException().or(UNKNOWN_VALUE)); taskMetadataBuilder.put(EventSubmitter.EVENT_TYPE, TASK_STATE); this.eventSubmitter.submit(TASK_STATE, taskMetadataBuilder.build()); } }
private TaskState createMockTaskState() { TaskState taskState = mock(TaskState.class, Mockito.RETURNS_SMART_NULLS); when(taskState.getWorkingState()).thenReturn(WorkUnitState.WorkingState.SUCCESSFUL); when(taskState.getTaskFailureException()).thenReturn(Optional.<String> absent()); return taskState; } }
@Override public void onTaskRunCompletion(Task task) { try { // Check the task state and handle task retry if task failed and // it has not reached the maximum number of retries WorkUnitState.WorkingState state = task.getTaskState().getWorkingState(); if (state == WorkUnitState.WorkingState.FAILED && task.getRetryCount() < this.maxTaskRetries) { this.taskExecutor.retry(task); return; } } catch (Throwable t) { LOG.error("Failed to process a task completion callback", t); } // Mark the completion of this task task.markTaskCompletion(); }
private void finalizeDatasetState(JobState.DatasetState datasetState, String datasetUrn) { for (TaskState taskState : datasetState.getTaskStates()) { // Backoff the actual high watermark to the low watermark for each task that has not been committed if (taskState.getWorkingState() != WorkUnitState.WorkingState.COMMITTED) { taskState.backoffActualHighWatermark(); if (this.jobContext.getJobCommitPolicy() == JobCommitPolicy.COMMIT_ON_FULL_SUCCESS) { // Determine the final dataset state based on the task states (post commit) and the job commit policy. // 1. If COMMIT_ON_FULL_SUCCESS is used, the processing of the dataset is considered failed if any // task for the dataset failed to be committed. // 2. Otherwise, the processing of the dataset is considered successful even if some tasks for the // dataset failed to be committed. datasetState.setState(JobState.RunningState.FAILED); Optional<String> taskStateException = taskState.getTaskFailureException(); log.warn("At least one task did not committed successfully. Setting dataset state to FAILED.", taskStateException.isPresent() ? taskStateException.get() : "Exception not set."); } } } datasetState.setId(datasetUrn); }
@Override public void onTaskCommitCompletion(Task task) { if (GobblinMetrics.isEnabled(task.getTaskState().getWorkunit())) { // Update record-level metrics after the task is done task.updateRecordMetrics(); task.updateByteMetrics(); } // Cancel the task state reporter associated with this task. The reporter might // not be found for the given task because the task fails before the task is // registered. So we need to make sure the reporter exists before calling cancel. if (this.scheduledReporters.containsKey(task.getTaskId())) { this.scheduledReporters.remove(task.getTaskId()).cancel(false); } LOGGER.info(String .format("Task %s completed in %dms with state %s", task.getTaskId(), task.getTaskState().getTaskDuration(), task.getTaskState().getWorkingState())); }
public void testLaunchJob() throws Exception { final ConcurrentHashMap<String, Boolean> runningMap = new ConcurrentHashMap<>(); // Normal job launcher final Properties properties = generateJobProperties(this.baseConfig, "1", "_1504201348470"); final GobblinHelixJobLauncher gobblinHelixJobLauncher = this.closer.register( new GobblinHelixJobLauncher(properties, this.helixManager, this.appWorkDir, ImmutableList.<Tag<?>>of(), runningMap, java.util.Optional.empty())); gobblinHelixJobLauncher.launchJob(null); final File jobOutputFile = getJobOutputFile(properties); Assert.assertTrue(jobOutputFile.exists()); Schema schema = new Schema.Parser().parse(TestHelper.SOURCE_SCHEMA); TestHelper.assertGenericRecords(jobOutputFile, schema); List<JobState.DatasetState> datasetStates = this.datasetStateStore.getAll(properties.getProperty(ConfigurationKeys.JOB_NAME_KEY), FsDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + FsDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX); Assert.assertEquals(datasetStates.size(), 1); JobState.DatasetState datasetState = datasetStates.get(0); Assert.assertEquals(datasetState.getCompletedTasks(), 1); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getTaskStates().size(), 1); Assert.assertEquals(datasetState.getTaskStates().get(0).getWorkingState(), WorkUnitState.WorkingState.COMMITTED); }
@Override public void onTaskCommitCompletion(Task task) { WorkUnit workUnit = task.getTaskState().getWorkunit(); if (GobblinMetrics.isEnabled(workUnit)) { task.updateRecordMetrics(); task.updateByteMetrics(); if (workUnit.getPropAsBoolean(ConfigurationKeys.MR_REPORT_METRICS_AS_COUNTERS_KEY, ConfigurationKeys.DEFAULT_MR_REPORT_METRICS_AS_COUNTERS)) { updateCounters(task); } } LOG.info(String .format("Task %s completed running in %dms with state %s", task.getTaskId(), task.getTaskState().getTaskDuration(), task.getTaskState().getWorkingState())); }
private void verifyJobState(int run) throws IOException { List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(JOB_NAME, "current.jst"); Assert.assertEquals(datasetStateList.size(), 1); JobState jobState = datasetStateList.get(0); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getTaskStates().size(), DummySource.NUM_WORK_UNITS); for (TaskState taskState : jobState.getTaskStates()) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getProp(FOO), BAR); // Check if the low watermark is properly kept track of int expectedLowWatermark = (run - 1) * DummySource.NUM_WORK_UNITS * DummySource.NUM_RECORDS_TO_EXTRACT_PER_EXTRACTOR + taskState.getPropAsInt(WORK_UNIT_INDEX_KEY) * DummySource.NUM_RECORDS_TO_EXTRACT_PER_EXTRACTOR + 1; Assert.assertEquals(taskState.getPropAsInt(ConfigurationKeys.WORK_UNIT_LOW_WATER_MARK_KEY), expectedLowWatermark); // Check if the high watermark is properly kept track of int expectedHighWatermark = expectedLowWatermark + DummySource.NUM_RECORDS_TO_EXTRACT_PER_EXTRACTOR - 1; Assert.assertEquals(taskState.getPropAsInt(ConfigurationKeys.WORK_UNIT_HIGH_WATER_MARK_KEY), expectedHighWatermark); Assert.assertEquals(taskState.getPropAsInt(LAST_READ_RECORD_KEY), expectedHighWatermark); } }
/** * Finalize a given {@link JobState.DatasetState} before committing the dataset. * * This method is thread-safe. */ private void finalizeDatasetStateBeforeCommit(JobState.DatasetState datasetState) { for (TaskState taskState : datasetState.getTaskStates()) { if (taskState.getWorkingState() != WorkUnitState.WorkingState.SUCCESSFUL && this.jobContext.getJobCommitPolicy() == JobCommitPolicy.COMMIT_ON_FULL_SUCCESS) { // The dataset state is set to FAILED if any task failed and COMMIT_ON_FULL_SUCCESS is used datasetState.setState(JobState.RunningState.FAILED); datasetState.incrementJobFailures(); return; } } datasetState.setState(JobState.RunningState.SUCCESSFUL); datasetState.setNoJobFailure(); }
public void runTestWithPullLimit(Properties jobProps, long limit) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally { closer.close(); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 4); Assert.assertEquals(datasetState.getJobFailures(), 0); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.EXTRACTOR_ROWS_EXTRACTED), limit); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_ROWS_WRITTEN), limit); } }
@Test(dependsOnMethods = "testPersistDatasetState") public void testGetDatasetState() throws IOException { JobState.DatasetState datasetState = zkDatasetStateStore.getLatestDatasetState(TEST_JOB_NAME, TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); Assert.assertEquals(datasetState.getCompletedTasks(), 3); for (int i = 0; i < datasetState.getCompletedTasks(); i++) { TaskState taskState = datasetState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = this.fsDatasetStateStore.get(TEST_JOB_NAME, FsDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + FsDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistDatasetState") public void testGetDatasetState() throws IOException { JobState.DatasetState datasetState = dbDatasetStateStore.getLatestDatasetState(TEST_JOB_NAME, TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); Assert.assertEquals(datasetState.getCompletedTasks(), 3); for (int i = 0; i < datasetState.getCompletedTasks(); i++) { TaskState taskState = datasetState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistDatasetState") public void testGetDatasetState() throws IOException { JobState.DatasetState datasetState = this.fsDatasetStateStore.getLatestDatasetState(TEST_JOB_NAME, TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); Assert.assertEquals(datasetState.getCompletedTasks(), 3); for (int i = 0; i < datasetState.getCompletedTasks(); i++) { TaskState taskState = datasetState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test public void testSetAndGet() { this.taskState.setId("Task-1"); this.taskState.setHighWaterMark(2000); this.startTime = System.currentTimeMillis(); this.taskState.setStartTime(this.startTime); this.taskState.setEndTime(this.startTime + 1000); this.taskState.setTaskDuration(1000); this.taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); this.taskState.setProp("foo", "bar"); Assert.assertEquals(this.taskState.getJobId(), "Job-1"); Assert.assertEquals(this.taskState.getTaskId(), "Task-1"); Assert.assertEquals(this.taskState.getId(), "Task-1"); Assert.assertEquals(this.taskState.getHighWaterMark(), 2000); Assert.assertEquals(this.taskState.getStartTime(), this.startTime); Assert.assertEquals(this.taskState.getEndTime(), this.startTime + 1000); Assert.assertEquals(this.taskState.getTaskDuration(), 1000); Assert.assertEquals(this.taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(this.taskState.getProp("foo"), "bar"); }
@Test public void testControlMessages() throws Exception { MyExtractor extractor = new MyExtractor(new StreamEntity[]{new RecordEnvelope<>("a"), new BasicTestControlMessage("1"), new RecordEnvelope<>("b"), new BasicTestControlMessage("2")}); MyConverter converter = new MyConverter(); MyDataWriter writer = new MyDataWriter(); Task task = setupTask(extractor, writer, converter); task.run(); task.commit(); Assert.assertEquals(task.getTaskState().getWorkingState(), WorkUnitState.WorkingState.SUCCESSFUL); Assert.assertEquals(converter.records, Lists.newArrayList("a", "b")); Assert.assertEquals(converter.messages, Lists.newArrayList(new BasicTestControlMessage("1"), new BasicTestControlMessage("2"))); Assert.assertEquals(writer.records, Lists.newArrayList("a", "b")); Assert.assertEquals(writer.messages, Lists.newArrayList(new BasicTestControlMessage("1"), new BasicTestControlMessage("2"))); }
@Test public void testFlushControlMessages() throws Exception { MyExtractor extractor = new MyExtractor(new StreamEntity[]{new RecordEnvelope<>("a"), FlushControlMessage.builder().flushReason("flush1").build(), new RecordEnvelope<>("b"), FlushControlMessage.builder().flushReason("flush2").build()}); MyConverter converter = new MyConverter(); MyFlushDataWriter writer = new MyFlushDataWriter(); Task task = setupTask(extractor, writer, converter); task.run(); task.commit(); Assert.assertEquals(task.getTaskState().getWorkingState(), WorkUnitState.WorkingState.SUCCESSFUL); Assert.assertEquals(converter.records, Lists.newArrayList("a", "b")); Assert.assertEquals(converter.messages, Lists.newArrayList( FlushControlMessage.builder().flushReason("flush1").build(), FlushControlMessage.builder().flushReason("flush2").build())); Assert.assertEquals(writer.records, Lists.newArrayList("a", "b")); Assert.assertEquals(writer.flush_messages, Lists.newArrayList("flush called", "flush called")); }