private static void migrateStateForJob(DatasetStateStore srcDatasetStateStore, DatasetStateStore dstDatasetStateStore, String jobName, boolean deleteFromSource) throws IOException { Map<String, JobState.DatasetState> map = srcDatasetStateStore.getLatestDatasetStatesByUrns(jobName); for (Map.Entry<String, JobState.DatasetState> entry : map.entrySet()) { dstDatasetStateStore.persistDatasetState(entry.getKey(), entry.getValue()); } if (deleteFromSource) { try { srcDatasetStateStore.delete(jobName); } catch (IOException ioe) { log.warn("The source state store has been deleted", ioe); } } }
@AfterClass public void tearDown() throws IOException { dbJobStateStore.delete(TEST_JOB_NAME); dbDatasetStateStore.delete(TEST_JOB_NAME); } }
this.datasetStateStore.delete(jobName); log.info("JobSpec {} deleted with statestore.", jobSpecUri); } else {
@Test(dependsOnMethods = "testGetPreviousDatasetStatesByUrns") public void testDeleteDatasetJobState() throws IOException { JobState.DatasetState datasetState = zkDatasetStateStore.get(TEST_JOB_NAME, TEST_DATASET_URN + "-" + zkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + zkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_DATASET_URN); Assert.assertNotNull(datasetState); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); zkDatasetStateStore.delete(TEST_JOB_NAME); datasetState = zkDatasetStateStore.get(TEST_JOB_NAME, TEST_DATASET_URN + "-" + zkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + zkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_DATASET_URN); Assert.assertNull(datasetState); }
@Test(dependsOnMethods = "testGetPreviousDatasetStatesByUrns") public void testDeleteDatasetJobState() throws IOException { JobState.DatasetState datasetState = dbDatasetStateStore.get(TEST_JOB_NAME, TEST_DATASET_URN + "-" + dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_DATASET_URN); Assert.assertNotNull(datasetState); Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID); dbDatasetStateStore.delete(TEST_JOB_NAME); datasetState = dbDatasetStateStore.get(TEST_JOB_NAME, TEST_DATASET_URN + "-" + dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_DATASET_URN); Assert.assertNull(datasetState); }
@BeforeClass public void setUp() throws Exception { ConfigBuilder configBuilder = ConfigBuilder.create(); testingServer = new TestingServer(-1); zkJobStateStore = new ZkStateStore<>(testingServer.getConnectString(), "/STATE_STORE/TEST", false, JobState.class); configBuilder.addPrimitive(ZkStateStoreConfigurationKeys.STATE_STORE_ZK_CONNECT_STRING_KEY, testingServer.getConnectString()); configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, "/STATE_STORE/TEST2"); ClassAliasResolver<DatasetStateStore.Factory> resolver = new ClassAliasResolver<>(DatasetStateStore.Factory.class); DatasetStateStore.Factory stateStoreFactory = resolver.resolveClass("zk").newInstance(); zkDatasetStateStore = stateStoreFactory.createStateStore(configBuilder.build()); // clear data that may have been left behind by a prior test run zkJobStateStore.delete(TEST_JOB_NAME); zkDatasetStateStore.delete(TEST_JOB_NAME); zkJobStateStore.delete(TEST_JOB_NAME2); zkDatasetStateStore.delete(TEST_JOB_NAME2); }
@BeforeClass public void setUp() throws Exception { testMetastoreDatabase = TestMetastoreDatabaseFactory.get(); String jdbcUrl = testMetastoreDatabase.getJdbcUrl(); ConfigBuilder configBuilder = ConfigBuilder.create(); BasicDataSource mySqlDs = new BasicDataSource(); mySqlDs.setDriverClassName(ConfigurationKeys.DEFAULT_STATE_STORE_DB_JDBC_DRIVER); mySqlDs.setDefaultAutoCommit(false); mySqlDs.setUrl(jdbcUrl); mySqlDs.setUsername(TEST_USER); mySqlDs.setPassword(TEST_PASSWORD); dbJobStateStore = new MysqlStateStore<>(mySqlDs, TEST_STATE_STORE, false, JobState.class); configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_DB_URL_KEY, jdbcUrl); configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_DB_USER_KEY, TEST_USER); configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, TEST_PASSWORD); ClassAliasResolver<DatasetStateStore.Factory> resolver = new ClassAliasResolver<>(DatasetStateStore.Factory.class); DatasetStateStore.Factory stateStoreFactory = resolver.resolveClass("mysql").newInstance(); dbDatasetStateStore = stateStoreFactory.createStateStore(configBuilder.build()); // clear data that may have been left behind by a prior test run dbJobStateStore.delete(TEST_JOB_NAME); dbDatasetStateStore.delete(TEST_JOB_NAME); dbJobStateStore.delete(TEST_JOB_NAME2); dbDatasetStateStore.delete(TEST_JOB_NAME2); }
@BeforeClass public void setUp() throws Exception { this.testMetastoreDatabase = TestMetastoreDatabaseFactory.get(); String jdbcUrl = this.testMetastoreDatabase.getJdbcUrl(); ConfigBuilder configBuilder = ConfigBuilder.create(); BasicDataSource mySqlDs = new BasicDataSource(); mySqlDs.setDriverClassName(ConfigurationKeys.DEFAULT_STATE_STORE_DB_JDBC_DRIVER); mySqlDs.setDefaultAutoCommit(false); mySqlDs.setUrl(jdbcUrl); mySqlDs.setUsername(TEST_USER); mySqlDs.setPassword(TEST_PASSWORD); this.dbJobStateStore = new MysqlStateStore<>(mySqlDs, TEST_STATE_STORE, false, JobState.class); configBuilder.addPrimitive("selection.timeBased.lookbackTime", "10m"); configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_TYPE_KEY, "mysql"); configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_DB_TABLE_KEY, TEST_STATE_STORE); configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_DB_URL_KEY, jdbcUrl); configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_DB_USER_KEY, TEST_USER); configBuilder.addPrimitive(ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, TEST_PASSWORD); ClassAliasResolver<DatasetStateStore.Factory> resolver = new ClassAliasResolver<>(DatasetStateStore.Factory.class); DatasetStateStore.Factory stateStoreFactory = resolver.resolveClass("mysql").newInstance(); this.config = configBuilder.build(); this.dbDatasetStateStore = stateStoreFactory.createStateStore(configBuilder.build()); // clear data that may have been left behind by a prior test run this.dbJobStateStore.delete(TEST_JOB_NAME1); this.dbDatasetStateStore.delete(TEST_JOB_NAME1); this.dbJobStateStore.delete(TEST_JOB_NAME2); this.dbDatasetStateStore.delete(TEST_JOB_NAME2); }
private static void migrateStateForJob(DatasetStateStore srcDatasetStateStore, DatasetStateStore dstDatasetStateStore, String jobName, boolean deleteFromSource) throws IOException { Map<String, JobState.DatasetState> map = srcDatasetStateStore.getLatestDatasetStatesByUrns(jobName); for (Map.Entry<String, JobState.DatasetState> entry : map.entrySet()) { dstDatasetStateStore.persistDatasetState(entry.getKey(), entry.getValue()); } if (deleteFromSource) { try { srcDatasetStateStore.delete(jobName); } catch (IOException ioe) { log.warn("The source state store has been deleted", ioe); } } }
this.datasetStateStore.delete(jobName); log.info("JobSpec {} deleted with statestore.", jobSpecUri); } else {