public StateStoreBasedWatermarkStorage(State taskState) { Preconditions.checkArgument(taskState != null); Preconditions.checkArgument(!taskState.getProp(ConfigurationKeys.JOB_NAME_KEY).isEmpty()); String watermarkStateStoreType = taskState.getProp(WATERMARK_STORAGE_TYPE_KEY, WATERMARK_STORAGE_TYPE_DEFAULT); ClassAliasResolver<StateStore.Factory> resolver = new ClassAliasResolver<>(StateStore.Factory.class); StateStore.Factory stateStoreFactory; try { stateStoreFactory = resolver.resolveClass(watermarkStateStoreType).newInstance(); } catch (ClassNotFoundException cnfe) { throw new RuntimeException(cnfe); } catch (InstantiationException ie) { throw new RuntimeException(ie); } catch (IllegalAccessException iae) { throw new RuntimeException(iae); } Config config = getStateStoreConfig(taskState); _stateStore = stateStoreFactory.createStateStore(config, CheckpointableWatermarkState.class); _storeName = WATERMARK_STORAGE_PREFIX + taskState.getProp(ConfigurationKeys.JOB_NAME_KEY); log.info("State Store directory configured as : {}", config.getString(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY)); log.info("Configured the StateStoreBasedWatermarkStorage with storeName: {}", _storeName); }
@Test public void testBackwardsCompat() throws IOException { // Tests with a state store that was saved before the WritableShim changes Config bwConfig = ConfigFactory.load(config); URL path = getClass().getResource("/backwardsCompatTestStore"); Assert.assertNotNull(path); bwConfig = bwConfig.withValue(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, ConfigValueFactory.fromAnyRef(path.toString())); StateStore<State> bwStateStore = stateStoreFactory.createStateStore(bwConfig, State.class); Assert.assertTrue(bwStateStore.exists("testStore", "testTable")); List<State> states = bwStateStore.getAll("testStore", "testTable"); Assert.assertEquals(states.size(), 3); Assert.assertEquals(states.get(0).getProp("k1"), "v1"); Assert.assertEquals(states.get(1).getProp("k2"), "v2"); Assert.assertEquals(states.get(2).getProp("k3"), "v3"); }
Config taskStateStoreConfig = getStateStoreConfig(config, taskStateOutputDir.toString(), taskStoreTable); taskStateStore = stateStoreFactory.createStateStore(taskStateStoreConfig, TaskState.class); wuStateStore = stateStoreFactory.createStateStore(wuStateStoreConfig, WorkUnit.class); mwuStateStore = stateStoreFactory.createStateStore(wuStateStoreConfig, MultiWorkUnit.class); Config jobStateStoreConfig = getStateStoreConfig(config, jobStateOutputDir.toString(), jobStateStoreTable); jobStateStore = stateStoreFactory.createStateStore(jobStateStoreConfig, JobState.class); } else { jobStateStore = null;
Config taskStateStoreConfig = getStateStoreConfig(config, taskStateOutputDir.toString(), taskStoreTable); taskStateStore = stateStoreFactory.createStateStore(taskStateStoreConfig, TaskState.class); wuStateStore = stateStoreFactory.createStateStore(wuStateStoreConfig, WorkUnit.class); mwuStateStore = stateStoreFactory.createStateStore(wuStateStoreConfig, MultiWorkUnit.class); Config jobStateStoreConfig = getStateStoreConfig(config, jobStateOutputDir.toString(), jobStateStoreTable); jobStateStore = stateStoreFactory.createStateStore(jobStateStoreConfig, JobState.class); } else { jobStateStore = null;
public StateStoreBasedWatermarkStorage(State taskState) { Preconditions.checkArgument(taskState != null); Preconditions.checkArgument(!taskState.getProp(ConfigurationKeys.JOB_NAME_KEY).isEmpty()); String watermarkStateStoreType = taskState.getProp(WATERMARK_STORAGE_TYPE_KEY, WATERMARK_STORAGE_TYPE_DEFAULT); ClassAliasResolver<StateStore.Factory> resolver = new ClassAliasResolver<>(StateStore.Factory.class); StateStore.Factory stateStoreFactory; try { stateStoreFactory = resolver.resolveClass(watermarkStateStoreType).newInstance(); } catch (ClassNotFoundException cnfe) { throw new RuntimeException(cnfe); } catch (InstantiationException ie) { throw new RuntimeException(ie); } catch (IllegalAccessException iae) { throw new RuntimeException(iae); } Config config = getStateStoreConfig(taskState); _stateStore = stateStoreFactory.createStateStore(config, CheckpointableWatermarkState.class); _storeName = WATERMARK_STORAGE_PREFIX + taskState.getProp(ConfigurationKeys.JOB_NAME_KEY); log.info("State Store directory configured as : {}", config.getString(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY)); log.info("Configured the StateStoreBasedWatermarkStorage with storeName: {}", _storeName); }
public HelixJobsMapping(Config sysConfig, URI fsUri, String rootDir) { String stateStoreType = ConfigUtils.getString(sysConfig, ConfigurationKeys.INTERMEDIATE_STATE_STORE_TYPE_KEY, ConfigUtils.getString(sysConfig, ConfigurationKeys.STATE_STORE_TYPE_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_TYPE)); ClassAliasResolver<StateStore.Factory> resolver = new ClassAliasResolver<>(StateStore.Factory.class); StateStore.Factory stateStoreFactory; try { stateStoreFactory = resolver.resolveClass(stateStoreType).newInstance(); } catch (ClassNotFoundException cnfe) { throw new RuntimeException(cnfe); } catch (InstantiationException ie) { throw new RuntimeException(ie); } catch (IllegalAccessException iae) { throw new RuntimeException(iae); } String dbTableKey = ConfigUtils.getString(sysConfig, JOBS_MAPPING_DB_TABLE_KEY, DEFAULT_JOBS_MAPPING_DB_TABLE_KEY_NAME); this.distributedStateStoreName = ConfigUtils.getString(sysConfig, DISTRIBUTED_STATE_STORE_NAME_KEY, DEFAULT_DISTRIBUTED_STATE_STORE_NAME); Config stateStoreJobConfig = sysConfig .withValue(ConfigurationKeys.STATE_STORE_FS_URI_KEY, ConfigValueFactory.fromAnyRef(fsUri.toString())) .withValue(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, ConfigValueFactory.fromAnyRef(rootDir)) .withValue(ConfigurationKeys.STATE_STORE_DB_TABLE_KEY, ConfigValueFactory.fromAnyRef(dbTableKey)); this.stateStore = stateStoreFactory.createStateStore(stateStoreJobConfig, State.class); }
@BeforeClass public void setUp() throws Exception { ClassAliasResolver<StateStore.Factory> resolver = new ClassAliasResolver<>(StateStore.Factory.class); stateStoreFactory = resolver.resolveClass("fs").newInstance(); config = ConfigFactory.empty().withValue(ConfigurationKeys.STATE_STORE_FS_URI_KEY, ConfigValueFactory.fromAnyRef("file:///")).withValue(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, ConfigValueFactory.fromAnyRef("metastore-test")).withValue("fs.permissions.umask-mode", ConfigValueFactory.fromAnyRef("022")); this.stateStore = stateStoreFactory.createStateStore(config, State.class); // cleanup in case files left behind by a prior run this.stateStore.delete("testStore"); this.stateStore.delete("testStore2"); }