public void deleteMapping (String jobName) throws IOException { this.stateStore.delete(this.distributedStateStoreName, jobName); }
@Test(dependsOnMethods = "testGetPreviousDatasetStatesByUrns") public void testDeleteJobState() throws IOException { JobState jobState = dbJobStateStore.get(TEST_JOB_NAME, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertNotNull(jobState); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); dbJobStateStore.delete(TEST_JOB_NAME); jobState = dbJobStateStore.get(TEST_JOB_NAME, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertNull(jobState); }
Iterable<CheckpointableWatermarkState> getAllCommittedWatermarks() throws IOException { return _stateStore.getAll(_storeName); }
@Override public Void call() throws Exception { TaskState taskState = taskStateStore.getAll(outputTaskStateDir.getName(), taskStateName).get(0); taskStateQueue.add(taskState); taskStateStore.delete(outputTaskStateDir.getName(), taskStateName); return null; } }, "Deserialize state for " + taskStateName);
@Test(dependsOnMethods = { "testPut" }) public void testCreateAlias() throws IOException { this.stateStore.createAlias("testStore", "testTable", "testTable1"); Assert.assertTrue(this.stateStore.exists("testStore", "testTable1")); }
if (taskStateStore.exists(jobId, taskId + AbstractJobLauncher.TASK_STATE_STORE_TABLE_SUFFIX)) { taskStateStore.delete(jobId, taskId + AbstractJobLauncher.TASK_STATE_STORE_TABLE_SUFFIX); for (Task task : tasks) { log.info("Writing task state for task " + task.getTaskId()); taskStateStore.put(task.getJobId(), task.getTaskId() + AbstractJobLauncher.TASK_STATE_STORE_TABLE_SUFFIX, task.getTaskState()); taskStateStore.put(task.getJobId(), task.getTaskId() + TASK_STATE_STORE_SUCCESS_MARKER_SUFFIX, task.getTaskState());
@Override public Void call() throws Exception { stateStore.put(storeName, fileName, workUnit); return null; } }, "Serialize state to store " + storeName + " file " + fileName);
private Optional<String> getId (String jobName, String idName) throws IOException { State state = this.stateStore.get(distributedStateStoreName, jobName, jobName); if (state == null) { return Optional.empty(); } String id = state.getProp(idName); return id == null? Optional.empty() : Optional.of(id); }
@Test public void testBackwardsCompat() throws IOException { // Tests with a state store that was saved before the WritableShim changes Config bwConfig = ConfigFactory.load(config); URL path = getClass().getResource("/backwardsCompatTestStore"); Assert.assertNotNull(path); bwConfig = bwConfig.withValue(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, ConfigValueFactory.fromAnyRef(path.toString())); StateStore<State> bwStateStore = stateStoreFactory.createStateStore(bwConfig, State.class); Assert.assertTrue(bwStateStore.exists("testStore", "testTable")); List<State> states = bwStateStore.getAll("testStore", "testTable"); Assert.assertEquals(states.size(), 3); Assert.assertEquals(states.get(0).getProp("k1"), "v1"); Assert.assertEquals(states.get(1).getProp("k2"), "v2"); Assert.assertEquals(states.get(2).getProp("k3"), "v3"); }
/** * Determine if the task executed successfully in a prior attempt by checkitn the task state store for the success * marker. * @param taskId task id to check * @return whether the task was processed successfully in a prior attempt */ private boolean taskSuccessfulInPriorAttempt(String taskId) { if (this.taskStateStoreOptional.isPresent()) { StateStore<TaskState> taskStateStore = this.taskStateStoreOptional.get(); // Delete the task state file for the task if it already exists. // This usually happens if the task is retried upon failure. try { if (taskStateStore.exists(jobId, taskId + TASK_STATE_STORE_SUCCESS_MARKER_SUFFIX)) { log.info("Skipping task {} that successfully executed in a prior attempt.", taskId); // skip tasks that executed successfully in a previous attempt return true; } } catch (IOException e) { // if an error while looking up the task state store then treat like it was not processed return false; } } return false; }
@Test public void testPut() throws IOException { List<State> states = Lists.newArrayList(); State state1 = new State(); state1.setId("s1"); state1.setProp("k1", "v1"); states.add(state1); State state2 = new State(); state2.setId("s2"); state2.setProp("k2", "v2"); states.add(state2); State state3 = new State(); state3.setId("s3"); state3.setProp("k3", "v3"); states.add(state3); Assert.assertFalse(this.stateStore.exists("testStore", "testTable")); this.stateStore.putAll("testStore", "testTable", states); Assert.assertTrue(this.stateStore.exists("testStore", "testTable")); // for testing of getStoreNames this.stateStore.putAll("testStore2", "testTable", states); }
@Test(dependsOnMethods = { "testGetAlias" }) public void testGetStoreNames() throws IOException { List<String> storeNames = this.stateStore.getStoreNames(Predicates.alwaysTrue()); Collections.sort(storeNames); Assert.assertTrue(storeNames.size() == 2); Assert.assertEquals(storeNames.get(0), "testStore"); Assert.assertEquals(storeNames.get(1), "testStore2"); }
List<String> taskStateNames = taskStateStore.getTableNames(outputTaskStateDir.getName(), new Predicate<String>() { @Override public boolean apply(String input) {
if (taskStateStore.exists(jobId, taskId + AbstractJobLauncher.TASK_STATE_STORE_TABLE_SUFFIX)) { taskStateStore.delete(jobId, taskId + AbstractJobLauncher.TASK_STATE_STORE_TABLE_SUFFIX); for (Task task : tasks) { log.info("Writing task state for task " + task.getTaskId()); taskStateStore.put(task.getJobId(), task.getTaskId() + AbstractJobLauncher.TASK_STATE_STORE_TABLE_SUFFIX, task.getTaskState()); taskStateStore.put(task.getJobId(), task.getTaskId() + TASK_STATE_STORE_SUCCESS_MARKER_SUFFIX, task.getTaskState());
@Override public void commitWatermarks(Iterable<CheckpointableWatermark> watermarks) throws IOException { for (CheckpointableWatermark watermark: watermarks) { String tableName = watermark.getSource(); _stateStore.put(_storeName, tableName, new CheckpointableWatermarkState(watermark, GSON)); } }
@Override public Void call() throws Exception { TaskState taskState = taskStateStore.getAll(outputTaskStateDir.getName(), taskStateName).get(0); taskStateQueue.add(taskState); taskStateStore.delete(outputTaskStateDir.getName(), taskStateName); return null; } }, "Deserialize state for " + taskStateName);
@Override public Map<String, CheckpointableWatermark> getCommittedWatermarks(Class<? extends CheckpointableWatermark> watermarkClass, Iterable<String> sourcePartitions) throws IOException { Map<String, CheckpointableWatermark> committed = new HashMap<String, CheckpointableWatermark>(); for (String sourcePartition: sourcePartitions) { CheckpointableWatermarkState watermarkState = _stateStore.get(_storeName, sourcePartition, sourcePartition); if (watermarkState != null) { CheckpointableWatermark watermark = GSON.fromJson(watermarkState.getProp(sourcePartition), watermarkClass); committed.put(sourcePartition, watermark); } } if (committed.isEmpty()) { log.warn("Didn't find any committed watermarks"); } return committed; }
/** * Determine if the task executed successfully in a prior attempt by checkitn the task state store for the success * marker. * @param taskId task id to check * @return whether the task was processed successfully in a prior attempt */ private boolean taskSuccessfulInPriorAttempt(String taskId) { if (this.taskStateStoreOptional.isPresent()) { StateStore<TaskState> taskStateStore = this.taskStateStoreOptional.get(); // Delete the task state file for the task if it already exists. // This usually happens if the task is retried upon failure. try { if (taskStateStore.exists(jobId, taskId + TASK_STATE_STORE_SUCCESS_MARKER_SUFFIX)) { log.info("Skipping task {} that successfully executed in a prior attempt.", taskId); // skip tasks that executed successfully in a previous attempt return true; } } catch (IOException e) { // if an error while looking up the task state store then treat like it was not processed return false; } } return false; }
@Test(dependsOnMethods = { "testGetDatasetState" }) public void testGetStoreNames() throws IOException { List<String> storeNames = this.zkJobStateStore.getStoreNames(Predicates.alwaysTrue()); Collections.sort(storeNames); Assert.assertTrue(storeNames.size() == 2); Assert.assertEquals(storeNames.get(0), TEST_JOB_NAME); Assert.assertEquals(storeNames.get(1), TEST_JOB_NAME2); storeNames = this.zkDatasetStateStore.getStoreNames(Predicates.alwaysTrue()); Collections.sort(storeNames); Assert.assertTrue(storeNames.size() == 2); Assert.assertEquals(storeNames.get(0), TEST_JOB_NAME); Assert.assertEquals(storeNames.get(1), TEST_JOB_NAME2); }