List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, "Dataset0-current.jst"); JobState.DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getTaskCount(), 1); TaskState taskState = datasetState.getTaskStates().get(0); JobState.DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getDatasetUrn(), "Dataset" + i); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 1); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getProp(ConfigurationKeys.DATASET_URN_KEY), "Dataset" + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED);
@Test(dependsOnMethods = "testGetStoreNames") public void testGetPreviousDatasetStatesByUrns() throws IOException { Map<String, JobState.DatasetState> datasetStatesByUrns = dbDatasetStateStore.getLatestDatasetStatesByUrns(TEST_JOB_NAME); Assert.assertEquals(datasetStatesByUrns.size(), 3); JobState.DatasetState datasetState = datasetStatesByUrns.get(TEST_DATASET_URN); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 1000); datasetState = datasetStatesByUrns.get(TEST_DATASET_URN2); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN2); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 2000); datasetState = datasetStatesByUrns.get(TEST_DATASET_URN_LOWER); Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN_LOWER); Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getStartTime(), this.startTime); Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000); Assert.assertEquals(datasetState.getDuration(), 3000); }
datasetStateMap.put(Integer.toString(i), new JobState.DatasetState());
if (datasetState.getState() == JobState.RunningState.FAILED) { jobState.setState(JobState.RunningState.FAILED); LOG.warn("At least one dataset state is FAILED. Setting job state to FAILED.");
@Override public Void call() throws Exception { if (this.datasetState.getState() == JobState.RunningState.COMMITTED) { log.info(this.datasetUrn + " have been committed."); return null; log.warn(String .format("Not committing dataset %s of job %s with commit policy %s and state %s", this.datasetUrn, this.jobContext.getJobId(), this.jobContext.getJobCommitPolicy(), this.datasetState.getState())); checkForUnpublishedWUHandling(this.datasetUrn, this.datasetState, dataPublisherClass, closer); throw new RuntimeException(String .format("Not committing dataset %s of job %s with commit policy %s and state %s", this.datasetUrn, this.jobContext.getJobId(), this.jobContext.getJobCommitPolicy(), this.datasetState.getState())); if (this.shouldCommitDataInJob) { log.info(String.format("Committing dataset %s of job %s with commit policy %s and state %s", this.datasetUrn, this.jobContext.getJobId(), this.jobContext.getJobCommitPolicy(), this.datasetState.getState())); this.datasetState.setState(JobState.RunningState.COMMITTED); } else { if (this.datasetState.getState() == JobState.RunningState.SUCCESSFUL) { this.datasetState.setState(JobState.RunningState.COMMITTED); if (commitSequenceBuilder.isPresent()) { buildAndExecuteCommitSequence(commitSequenceBuilder.get(), datasetState, datasetUrn); datasetState.setState(JobState.RunningState.COMMITTED); } else if (canPersistStates) { persistDatasetState(datasetUrn, datasetState);
Object writable = reader.getValueClass() == JobState.class ? new JobState() : new JobState.DatasetState(); if (writable instanceof JobState.DatasetState) { states.add((JobState.DatasetState) writable); writable = new JobState.DatasetState(); } else { states.add(((JobState) writable).newDatasetState(true));
private void maySubmitLineageEvent(JobState.DatasetState datasetState) { Collection<TaskState> allStates = datasetState.getTaskStates(); Collection<TaskState> states = Lists.newArrayList(); // Filter out failed states or states that don't have lineage info for (TaskState state : allStates) { if (state.getWorkingState() == WorkUnitState.WorkingState.COMMITTED && LineageInfo.hasLineageInfo(state)) { states.add(state); } } if (states.size() == 0) { log.info("Will not submit lineage events as no state contains lineage info"); return; } try { if (StringUtils.isEmpty(datasetUrn)) { // This dataset may contain different kinds of LineageEvent for (Map.Entry<String, Collection<TaskState>> entry : aggregateByLineageEvent(states).entrySet()) { submitLineageEvent(entry.getKey(), entry.getValue()); } } else { submitLineageEvent(datasetUrn, states); } } finally { // Purge lineage info from all states for (TaskState taskState : allStates) { LineageInfo.purgeLineageInfo(taskState); } } }
deserializeConf)) { Object writable = reader.getValueClass() == JobState.class ? new JobState() : new JobState.DatasetState();
/** * Persist a given {@link JobState.DatasetState}. * * @param datasetUrn the dataset URN * @param datasetState the {@link JobState.DatasetState} to persist * @throws IOException if there's something wrong persisting the {@link JobState.DatasetState} */ public void persistDatasetState(String datasetUrn, JobState.DatasetState datasetState) throws IOException { String jobName = datasetState.getJobName(); String jobId = datasetState.getJobId(); datasetUrn = CharMatcher.is(':').replaceFrom(datasetUrn, '.'); String datasetStatestoreName = sanitizeDatasetStatestoreNameFromDatasetURN(jobName, datasetUrn); String tableName = Strings.isNullOrEmpty(datasetUrn) ? sanitizeJobId(jobId) + DATASET_STATE_STORE_TABLE_SUFFIX : datasetStatestoreName + "-" + sanitizeJobId(jobId) + DATASET_STATE_STORE_TABLE_SUFFIX; LOGGER.info("Persisting " + tableName + " to the job state store"); put(jobName, tableName, datasetState); createAlias(jobName, tableName, getAliasName(datasetStatestoreName)); Path originalDatasetUrnPath = new Path(new Path(this.storeRootDir, jobName), getAliasName(datasetUrn)); // This should only happen for the first time. if (!Strings.isNullOrEmpty(datasetUrn) && !datasetStatestoreName.equals(datasetUrn) && this.fs .exists(originalDatasetUrnPath)) { LOGGER.info("Removing previous datasetUrn path: " + originalDatasetUrnPath); fs.delete(originalDatasetUrnPath, true); } }
/** * Create a {@link Map} from dataset URNs (as being specified by {@link ConfigurationKeys#DATASET_URN_KEY} to * {@link DatasetState} objects that represent the dataset states and store {@link TaskState}s corresponding * to the datasets. * * <p> * {@link TaskState}s that do not have {@link ConfigurationKeys#DATASET_URN_KEY} set will be added to * the dataset state belonging to {@link ConfigurationKeys#DEFAULT_DATASET_URN}. * </p> * * @return a {@link Map} from dataset URNs to {@link DatasetState}s representing the dataset states */ public Map<String, DatasetState> createDatasetStatesByUrns() { Map<String, DatasetState> datasetStatesByUrns = Maps.newHashMap(); for (TaskState taskState : this.taskStates.values()) { String datasetUrn = createDatasetUrn(datasetStatesByUrns, taskState); datasetStatesByUrns.get(datasetUrn).incrementTaskCount(); datasetStatesByUrns.get(datasetUrn).addTaskState(taskState); } for (TaskState taskState : this.skippedTaskStates.values()) { String datasetUrn = createDatasetUrn(datasetStatesByUrns, taskState); datasetStatesByUrns.get(datasetUrn).addSkippedTaskState(taskState); } return ImmutableMap.copyOf(datasetStatesByUrns); }
/** * Check if it is OK to commit the output data of a dataset. * * <p> * A dataset can be committed if and only if any of the following conditions is satisfied: * * <ul> * <li>The {@link JobCommitPolicy#COMMIT_ON_PARTIAL_SUCCESS} policy is used.</li> * <li>The {@link JobCommitPolicy#COMMIT_SUCCESSFUL_TASKS} policy is used.</li> * <li>The {@link JobCommitPolicy#COMMIT_ON_FULL_SUCCESS} policy is used and all of the tasks succeed.</li> * </ul> * </p> * This method is thread-safe. */ private boolean canCommitDataset(JobState.DatasetState datasetState) { // Only commit a dataset if 1) COMMIT_ON_PARTIAL_SUCCESS is used, or 2) // COMMIT_ON_FULL_SUCCESS is used and all of the tasks of the dataset have succeeded. return this.jobContext.getJobCommitPolicy() == JobCommitPolicy.COMMIT_ON_PARTIAL_SUCCESS || this.jobContext.getJobCommitPolicy() == JobCommitPolicy.COMMIT_SUCCESSFUL_TASKS || ( this.jobContext.getJobCommitPolicy() == JobCommitPolicy.COMMIT_ON_FULL_SUCCESS && datasetState.getState() == JobState.RunningState.SUCCESSFUL); }
/** * Get a {@link Map} from dataset URNs to the latest {@link JobState.DatasetState}s. * * @param jobName the job name * @return a {@link Map} from dataset URNs to the latest {@link JobState.DatasetState}s * @throws IOException if there's something wrong reading the {@link JobState.DatasetState}s */ public Map<String, JobState.DatasetState> getLatestDatasetStatesByUrns(String jobName) throws IOException { List<JobState.DatasetState> previousDatasetStates = getAll(jobName, "%" + CURRENT_DATASET_STATE_FILE_SUFFIX + DATASET_STATE_STORE_TABLE_SUFFIX, true); Map<String, JobState.DatasetState> datasetStatesByUrns = Maps.newHashMap(); for (JobState.DatasetState previousDatasetState : previousDatasetStates) { datasetStatesByUrns.put(previousDatasetState.getDatasetUrn(), previousDatasetState); } // The dataset (job) state from the deprecated "current.jst" will be read even though // the job has transitioned to the new dataset-based mechanism if (datasetStatesByUrns.size() > 1) { datasetStatesByUrns.remove(ConfigurationKeys.DEFAULT_DATASET_URN); } return datasetStatesByUrns; }
private void finalizeDatasetState(JobState.DatasetState datasetState, String datasetUrn) { for (TaskState taskState : datasetState.getTaskStates()) { // Backoff the actual high watermark to the low watermark for each task that has not been committed if (taskState.getWorkingState() != WorkUnitState.WorkingState.COMMITTED) { taskState.backoffActualHighWatermark(); if (this.jobContext.getJobCommitPolicy() == JobCommitPolicy.COMMIT_ON_FULL_SUCCESS) { // Determine the final dataset state based on the task states (post commit) and the job commit policy. // 1. If COMMIT_ON_FULL_SUCCESS is used, the processing of the dataset is considered failed if any // task for the dataset failed to be committed. // 2. Otherwise, the processing of the dataset is considered successful even if some tasks for the // dataset failed to be committed. datasetState.setState(JobState.RunningState.FAILED); Optional<String> taskStateException = taskState.getTaskFailureException(); log.warn("At least one task did not committed successfully. Setting dataset state to FAILED.", taskStateException.isPresent() ? taskStateException.get() : "Exception not set."); } } } datasetState.setId(datasetUrn); }
/** * Create a new {@link JobState.DatasetState} based on this {@link JobState} instance. * * @param fullCopy whether to do a full copy of this {@link JobState} instance * @return a new {@link JobState.DatasetState} object */ public DatasetState newDatasetState(boolean fullCopy) { DatasetState datasetState = new DatasetState(this.jobName, this.jobId); datasetState.setStartTime(this.startTime); datasetState.setEndTime(this.endTime); datasetState.setDuration(this.duration); if (fullCopy) { datasetState.setState(this.state); datasetState.setTaskCount(this.taskCount); datasetState.addTaskStates(this.taskStates.values()); datasetState.addSkippedTaskStates(this.skippedTaskStates.values()); } return datasetState; }
/** * Persist a given {@link JobState.DatasetState}. * * @param datasetUrn the dataset URN * @param datasetState the {@link JobState.DatasetState} to persist * @throws IOException if there's something wrong persisting the {@link JobState.DatasetState} */ public void persistDatasetState(String datasetUrn, JobState.DatasetState datasetState) throws IOException { String jobName = datasetState.getJobName(); String jobId = datasetState.getJobId(); datasetUrn = CharMatcher.is(':').replaceFrom(datasetUrn, '.'); String tableName = Strings.isNullOrEmpty(datasetUrn) ? jobId + DATASET_STATE_STORE_TABLE_SUFFIX : datasetUrn + "-" + jobId + DATASET_STATE_STORE_TABLE_SUFFIX; LOGGER.info("Persisting " + tableName + " to the job state store"); put(jobName, tableName, datasetState); createAlias(jobName, tableName, getAliasName(datasetUrn)); }
@SuppressWarnings("unchecked") private Optional<CommitSequence.Builder> generateCommitSequenceBuilder(JobState.DatasetState datasetState, Collection<TaskState> taskStates) { try (Closer closer = Closer.create()) { Class<? extends CommitSequencePublisher> dataPublisherClass = (Class<? extends CommitSequencePublisher>) Class .forName(datasetState .getProp(ConfigurationKeys.DATA_PUBLISHER_TYPE, ConfigurationKeys.DEFAULT_DATA_PUBLISHER_TYPE)); CommitSequencePublisher publisher = (CommitSequencePublisher) closer .register(DataPublisher.getInstance(dataPublisherClass, this.jobContext.getJobState())); publisher.publish(taskStates); return publisher.getCommitSequenceBuilder(); } catch (Throwable t) { log.error("Failed to generate commit sequence", t); setTaskFailureException(datasetState.getTaskStates(), t); throw Throwables.propagate(t); } }
/** * Finalize a given {@link JobState.DatasetState} before committing the dataset. * * This method is thread-safe. */ private void finalizeDatasetStateBeforeCommit(JobState.DatasetState datasetState) { for (TaskState taskState : datasetState.getTaskStates()) { if (taskState.getWorkingState() != WorkUnitState.WorkingState.SUCCESSFUL && this.jobContext.getJobCommitPolicy() == JobCommitPolicy.COMMIT_ON_FULL_SUCCESS) { // The dataset state is set to FAILED if any task failed and COMMIT_ON_FULL_SUCCESS is used datasetState.setState(JobState.RunningState.FAILED); datasetState.incrementJobFailures(); return; } } datasetState.setState(JobState.RunningState.SUCCESSFUL); datasetState.setNoJobFailure(); }
@Override public boolean isCompleted() throws IOException { Preconditions.checkNotNull(this.datasetState); return this.datasetState .equals(getDatasetStateStore().getLatestDatasetState(this.datasetState.getJobName(), this.datasetUrn)); }
private ListMultimap<TaskFactoryWrapper, TaskState> groupByTaskFactory(JobState.DatasetState datasetState) { ListMultimap<TaskFactoryWrapper, TaskState> groupsMap = ArrayListMultimap.create(); for (TaskState taskState : datasetState.getTaskStates()) { groupsMap.put(new TaskFactoryWrapper(TaskUtils.getTaskFactory(taskState).orNull()), taskState); } return groupsMap; }
private String createDatasetUrn(Map<String, DatasetState> datasetStatesByUrns, TaskState taskState) { String datasetUrn = taskState.getProp(ConfigurationKeys.DATASET_URN_KEY, ConfigurationKeys.DEFAULT_DATASET_URN); if (!datasetStatesByUrns.containsKey(datasetUrn)) { DatasetState datasetState = newDatasetState(false); datasetState.setDatasetUrn(datasetUrn); datasetStatesByUrns.put(datasetUrn, datasetState); } return datasetUrn; }