/** * Write a summary to the json document * * @param jsonWriter a {@link com.google.gson.stream.JsonWriter} * used to write the json document */ protected void writeStateSummary(JsonWriter jsonWriter) throws IOException { jsonWriter.name("job name").value(this.getJobName()).name("job id").value(this.getJobId()).name("job state") .value(this.getState().name()).name("start time").value(this.getStartTime()).name("end time") .value(this.getEndTime()).name("duration").value(this.getDuration()).name("tasks").value(this.getTaskCount()) .name("completed tasks").value(this.getCompletedTasks()); }
/** * Submits an event for the given {@link JobState}. */ private void submitJobStateEvent(JobState jobState) { ImmutableMap.Builder<String, String> jobMetadataBuilder = new ImmutableMap.Builder<>(); jobMetadataBuilder.put(METADATA_JOB_ID, jobState.getJobId()); jobMetadataBuilder.put(METADATA_JOB_NAME, jobState.getJobName()); jobMetadataBuilder.put(METADATA_JOB_START_TIME, Long.toString(jobState.getStartTime())); jobMetadataBuilder.put(METADATA_JOB_END_TIME, Long.toString(jobState.getEndTime())); jobMetadataBuilder.put(METADATA_JOB_STATE, jobState.getState().toString()); jobMetadataBuilder.put(METADATA_JOB_LAUNCHED_TASKS, Integer.toString(jobState.getTaskCount())); jobMetadataBuilder.put(METADATA_JOB_COMPLETED_TASKS, Integer.toString(jobState.getCompletedTasks())); jobMetadataBuilder.put(METADATA_JOB_LAUNCHER_TYPE, jobState.getLauncherType().toString()); jobMetadataBuilder.put(METADATA_JOB_TRACKING_URL, jobState.getTrackingURL().or(UNKNOWN_VALUE)); jobMetadataBuilder.put(EventSubmitter.EVENT_TYPE, JOB_STATE); this.eventSubmitter.submit(JOB_STATE, jobMetadataBuilder.build()); }
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = zkJobStateStore.get(TEST_JOB_NAME, zkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + zkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = this.fsDatasetStateStore.get(TEST_JOB_NAME, FsDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + FsDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
public void runTestWithCommitSuccessfulTasksPolicy(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); jobProps.setProperty(ConfigurationKeys.PUBLISH_DATA_AT_JOB_LEVEL, Boolean.FALSE.toString()); jobProps.setProperty(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "successful"); jobProps.setProperty(ConfigurationKeys.SOURCE_CLASS_KEY, TestSourceWithFaultyExtractor.class.getName()); jobProps.setProperty(ConfigurationKeys.MAX_TASK_RETRIES_KEY, "0"); Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally { closer.close(); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); JobState jobState = datasetStateList.get(0); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getCompletedTasks(), 4); for (TaskState taskState : jobState.getTaskStates()) { if (taskState.getTaskId().endsWith("0")) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.FAILED); } else { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS); } } }
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = dbJobStateStore.get(TEST_JOB_NAME, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getProp("foo"), "bar"); Assert.assertNotEquals(jobState.getProp("lower"), "case"); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } jobState = dbJobStateStore.get(TEST_JOB_NAME_LOWER, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME_LOWER); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getProp("foo"), "bar"); Assert.assertEquals(jobState.getProp("lower"), "case"); }
Assert.assertEquals(jobState.getCompletedTasks(), 3); for (TaskState taskState : jobState.getTaskStates()) { if (taskState.getTaskId().endsWith(skippedTaskSuffix)) {
for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), "TestJob-1"); Assert.assertEquals(sortedDatasetUrns.size(), jobState.getCompletedTasks()); Map<String, JobState.DatasetState> datasetStatesByUrns = jobState.createDatasetStatesByUrns(); int index = 0;
jobExecutionInfo.setState(JobStateEnum.valueOf(this.state.name())); jobExecutionInfo.setLaunchedTasks(this.taskCount); jobExecutionInfo.setCompletedTasks(this.getCompletedTasks()); jobExecutionInfo.setLauncherType(getLauncherType()); if (getTrackingURL().isPresent()) {
/** * Write a summary to the json document * * @param jsonWriter a {@link com.google.gson.stream.JsonWriter} * used to write the json document */ protected void writeStateSummary(JsonWriter jsonWriter) throws IOException { jsonWriter.name("job name").value(this.getJobName()).name("job id").value(this.getJobId()).name("job state") .value(this.getState().name()).name("start time").value(this.getStartTime()).name("end time") .value(this.getEndTime()).name("duration").value(this.getDuration()).name("tasks").value(this.getTaskCount()) .name("completed tasks").value(this.getCompletedTasks()); }
/** * Submits an event for the given {@link JobState}. */ private void submitJobStateEvent(JobState jobState) { ImmutableMap.Builder<String, String> jobMetadataBuilder = new ImmutableMap.Builder<>(); jobMetadataBuilder.put(METADATA_JOB_ID, jobState.getJobId()); jobMetadataBuilder.put(METADATA_JOB_NAME, jobState.getJobName()); jobMetadataBuilder.put(METADATA_JOB_START_TIME, Long.toString(jobState.getStartTime())); jobMetadataBuilder.put(METADATA_JOB_END_TIME, Long.toString(jobState.getEndTime())); jobMetadataBuilder.put(METADATA_JOB_STATE, jobState.getState().toString()); jobMetadataBuilder.put(METADATA_JOB_LAUNCHED_TASKS, Integer.toString(jobState.getTaskCount())); jobMetadataBuilder.put(METADATA_JOB_COMPLETED_TASKS, Integer.toString(jobState.getCompletedTasks())); jobMetadataBuilder.put(METADATA_JOB_LAUNCHER_TYPE, jobState.getLauncherType().toString()); jobMetadataBuilder.put(METADATA_JOB_TRACKING_URL, jobState.getTrackingURL().or(UNKNOWN_VALUE)); jobMetadataBuilder.put(EventSubmitter.EVENT_TYPE, JOB_STATE); this.eventSubmitter.submit(JOB_STATE, jobMetadataBuilder.build()); }
jobExecutionInfo.setState(JobStateEnum.valueOf(this.state.name())); jobExecutionInfo.setLaunchedTasks(this.taskCount); jobExecutionInfo.setCompletedTasks(this.getCompletedTasks()); jobExecutionInfo.setLauncherType(getLauncherType()); if (getTrackingURL().isPresent()) {