private void failTask(Throwable t) { LOG.error(String.format("Task %s failed", this.taskId), t); this.taskState.setWorkingState(WorkUnitState.WorkingState.FAILED); this.taskState.setProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY, Throwables.getStackTraceAsString(t)); // Send task failure event FailureEventBuilder failureEvent = new FailureEventBuilder(FAILED_TASK_EVENT); failureEvent.setRootCause(t); failureEvent.addMetadata(TASK_STATE, this.taskState.toString()); failureEvent.submit(taskContext.getTaskMetrics().getMetricContext()); }
this.startTime = System.currentTimeMillis(); this.taskState.setStartTime(startTime); this.taskState.setWorkingState(WorkUnitState.WorkingState.RUNNING);
this.taskState.setWorkingState(WorkUnitState.WorkingState.SUCCESSFUL); this.taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED);
@Test public void testSetAndGet() { this.taskState.setId("Task-1"); this.taskState.setHighWaterMark(2000); this.startTime = System.currentTimeMillis(); this.taskState.setStartTime(this.startTime); this.taskState.setEndTime(this.startTime + 1000); this.taskState.setTaskDuration(1000); this.taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); this.taskState.setProp("foo", "bar"); Assert.assertEquals(this.taskState.getJobId(), "Job-1"); Assert.assertEquals(this.taskState.getTaskId(), "Task-1"); Assert.assertEquals(this.taskState.getId(), "Task-1"); Assert.assertEquals(this.taskState.getHighWaterMark(), 2000); Assert.assertEquals(this.taskState.getStartTime(), this.startTime); Assert.assertEquals(this.taskState.getEndTime(), this.startTime + 1000); Assert.assertEquals(this.taskState.getTaskDuration(), 1000); Assert.assertEquals(this.taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(this.taskState.getProp("foo"), "bar"); }
@Override public TaskState getTaskState() { TaskState taskState = this.taskContext.getTaskState(); taskState.setTaskId(getTaskId()); taskState.setJobId(getJobId()); taskState.setWorkingState(getWorkingState()); taskState.addAll(getExecutionMetadata()); taskState.addAll(getPersistentState()); return taskState; }
public void persistDatasetState(List<Dataset> datasets, List<LongWatermark> watermarks, String jobName) throws IOException { Preconditions.checkArgument(datasets.size() >= 2); for (int i = 0; i < datasets.size(); i++) { String datasetUrn = datasets.get(i).getUrn(); JobState.DatasetState datasetState = new JobState.DatasetState(jobName, TEST_JOB_ID); datasetState.setDatasetUrn(datasetUrn); datasetState.setState(JobState.RunningState.COMMITTED); datasetState.setId(datasetUrn); datasetState.setStartTime(this.startTime); datasetState.setEndTime(this.startTime + 1000); datasetState.setDuration(1000); TaskState taskState = new TaskState(); taskState.setJobId(TEST_JOB_ID); taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); if (i < datasets.size() - 1) { taskState.setActualHighWatermark(watermarks.get(i)); } datasetState.addTaskState(taskState); this.fsDatasetStateStore.persistDatasetState(datasetUrn, datasetState); } }
taskState.setEndTime(this.startTime + 1000); taskState.setTaskDuration(1000); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); taskState.setProp("foo", "bar");
@Test(dependsOnMethods = "testGetJobState") public void testPersistDatasetState() throws IOException { JobState.DatasetState datasetState = new JobState.DatasetState(TEST_JOB_NAME, TEST_JOB_ID); datasetState.setDatasetUrn(TEST_DATASET_URN); datasetState.setState(JobState.RunningState.COMMITTED); datasetState.setId(TEST_DATASET_URN); datasetState.setStartTime(this.startTime); datasetState.setEndTime(this.startTime + 1000); datasetState.setDuration(1000); for (int i = 0; i < 3; i++) { TaskState taskState = new TaskState(); taskState.setJobId(TEST_JOB_ID); taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); datasetState.addTaskState(taskState); } this.fsDatasetStateStore.persistDatasetState(TEST_DATASET_URN, datasetState); }
@Test public void testPersistJobState() throws IOException { JobState jobState = new JobState(TEST_JOB_NAME, TEST_JOB_ID); jobState.setId(TEST_JOB_ID); jobState.setProp("foo", "bar"); jobState.setState(JobState.RunningState.COMMITTED); jobState.setStartTime(this.startTime); jobState.setEndTime(this.startTime + 1000); jobState.setDuration(1000); for (int i = 0; i < 3; i++) { TaskState taskState = new TaskState(); taskState.setJobId(TEST_JOB_ID); taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); jobState.addTaskState(taskState); } this.fsJobStateStore.put(TEST_JOB_NAME, FsDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + FsDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); }
taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); jobState.addTaskState(taskState);
@Test public void testPersistJobState() throws IOException { JobState jobState = new JobState(TEST_JOB_NAME, TEST_JOB_ID); jobState.setId(TEST_JOB_ID); jobState.setProp("foo", "bar"); jobState.setState(JobState.RunningState.COMMITTED); jobState.setStartTime(this.startTime); jobState.setEndTime(this.startTime + 1000); jobState.setDuration(1000); for (int i = 0; i < 3; i++) { TaskState taskState = new TaskState(); taskState.setJobId(TEST_JOB_ID); taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); jobState.addTaskState(taskState); } zkJobStateStore.put(TEST_JOB_NAME, ZkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + ZkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); // second job name for testing getting store names in a later test case jobState.setJobName(TEST_JOB_NAME2); zkJobStateStore.put(TEST_JOB_NAME2, ZkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + ZkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); }
taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); datasetState.addTaskState(taskState);
@Test(dependsOnMethods = "testGetJobState") public void testPersistDatasetState() throws IOException { JobState.DatasetState datasetState = new JobState.DatasetState(TEST_JOB_NAME, TEST_JOB_ID); datasetState.setDatasetUrn(TEST_DATASET_URN); datasetState.setState(JobState.RunningState.COMMITTED); datasetState.setId(TEST_DATASET_URN); datasetState.setStartTime(this.startTime); datasetState.setEndTime(this.startTime + 1000); datasetState.setDuration(1000); for (int i = 0; i < 3; i++) { TaskState taskState = new TaskState(); taskState.setJobId(TEST_JOB_ID); taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); datasetState.addTaskState(taskState); } zkDatasetStateStore.persistDatasetState(TEST_DATASET_URN, datasetState); // persist a second dataset state to test that retrieval of multiple dataset states works datasetState.setDatasetUrn(TEST_DATASET_URN2); datasetState.setId(TEST_DATASET_URN2); datasetState.setDuration(2000); zkDatasetStateStore.persistDatasetState(TEST_DATASET_URN2, datasetState); // second job name for testing getting store names in a later test case datasetState.setJobName(TEST_JOB_NAME2); zkDatasetStateStore.persistDatasetState(TEST_DATASET_URN2, datasetState); }
taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); jobState.addTaskState(taskState);
taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); datasetState.addTaskState(taskState);
private void failTask(Throwable t) { LOG.error(String.format("Task %s failed", this.taskId), t); this.taskState.setWorkingState(WorkUnitState.WorkingState.FAILED); this.taskState.setProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY, Throwables.getStackTraceAsString(t)); // Send task failure event FailureEventBuilder failureEvent = new FailureEventBuilder(FAILED_TASK_EVENT); failureEvent.setRootCause(t); failureEvent.addMetadata(TASK_STATE, this.taskState.toString()); failureEvent.submit(taskContext.getTaskMetrics().getMetricContext()); }
this.startTime = System.currentTimeMillis(); this.taskState.setStartTime(startTime); this.taskState.setWorkingState(WorkUnitState.WorkingState.RUNNING);
this.taskState.setWorkingState(WorkUnitState.WorkingState.SUCCESSFUL); this.taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED);
@Override public TaskState getTaskState() { TaskState taskState = this.taskContext.getTaskState(); taskState.setTaskId(getTaskId()); taskState.setJobId(getJobId()); taskState.setWorkingState(getWorkingState()); taskState.addAll(getExecutionMetadata()); taskState.addAll(getPersistentState()); return taskState; }