public JavaStorageConfig getStorageConfigs() { HashMap<String, String> configs = new HashMap<>(); configs.put(String.format(StorageConfig.FACTORY(), this.getStoreName()), this.getStoreFactory()); configs.put(String.format(StorageConfig.CHANGELOG_STREAM(), this.getStoreName()), this.getChangelogStream()); configs.putAll(this.getOtherProperties()); return new JavaStorageConfig(new MapConfig(configs)); }
storeConfig.put(String.format(StorageConfig.KEY_SERDE(), tableSpec.getId()), keySerde); storeConfig.put(String.format(StorageConfig.MSG_SERDE(), tableSpec.getId()), valueSerde); storeConfig.put(String.format(StorageConfig.CHANGELOG_STREAM(), tableSpec.getId()), changelogStream); storeConfig.put(String.format(StorageConfig.CHANGELOG_REPLICATION_FACTOR(), tableSpec.getId()), changelogReplicationFactor);
@Override public void onBecomingLeader() { LOG.info("ZkJobCoordinator::onBecomeLeader - I became the leader"); metrics.isLeader.set(true); zkUtils.subscribeToProcessorChange(new ProcessorChangeHandler(zkUtils)); if (!new StorageConfig(config).hasDurableStores()) { // 1. Stop if there's a existing StreamPartitionCountMonitor running. if (streamPartitionCountMonitor != null) { streamPartitionCountMonitor.stop(); } // 2. Start a new instance of StreamPartitionCountMonitor. streamPartitionCountMonitor = getPartitionCountMonitor(); streamPartitionCountMonitor.start(); } debounceTimer.scheduleAfterDebounceTime(ON_PROCESSOR_CHANGE, debounceTimeMs, ZkJobCoordinator.this::doOnProcessorChange); } }
/** * Directory loggedStoreDir associated with the logged store storeName is determined to be valid * if all of the following conditions are true. * a) If the store has to be persisted to disk. * b) If there is a valid offset file associated with the logged store. * c) If the logged store has not gone stale. * * @return true if the logged store is valid, false otherwise. */ private boolean isLoggedStoreValid(String storeName, File loggedStoreDir) { long changeLogDeleteRetentionInMs = StorageConfig.DEFAULT_CHANGELOG_DELETE_RETENTION_MS(); if (new StorageConfig(config).getChangeLogDeleteRetentionsInMs().get(storeName).isDefined()) { changeLogDeleteRetentionInMs = (long) new StorageConfig(config).getChangeLogDeleteRetentionsInMs().get(storeName).get(); } return this.taskStores.get(storeName).getStoreProperties().isPersistedToDisk() && StorageManagerUtil.isOffsetFileValid(loggedStoreDir, OFFSET_FILE_NAME) && !StorageManagerUtil.isStaleStore( loggedStoreDir, OFFSET_FILE_NAME, changeLogDeleteRetentionInMs, clock.currentTimeMillis()); }
Map<TaskName, TaskInstanceCollector> taskInstanceCollectors, StorageEngineFactory.StoreMode storeMode) { StorageConfig storageConfig = new StorageConfig(config); this.storeDirectoryPaths.add(storeDirectory.toPath()); if (storageConfig.getStorageKeySerde(storeName).isEmpty()) { throw new SamzaException("No key serde defined for store: " + storeName); Serde keySerde = serdes.get(storageConfig.getStorageKeySerde(storeName).get()); if (keySerde == null) { throw new SamzaException( "StorageKeySerde: No class defined for serde: " + storageConfig.getStorageKeySerde(storeName)); if (storageConfig.getStorageMsgSerde(storeName).isEmpty()) { throw new SamzaException("No msg serde defined for store: " + storeName); Serde messageSerde = serdes.get(storageConfig.getStorageMsgSerde(storeName).get()); if (messageSerde == null) { throw new SamzaException( "StorageMsgSerde: No class defined for serde: " + storageConfig.getStorageMsgSerde(storeName));
StorageConfig storageConfig = new StorageConfig(prevConfig); for (String store : JavaConversions.asJavaCollection(storageConfig.getStoreNames())) { String changelog = storageConfig.getChangelogStream(store) .getOrElse(defaultValue(null)); if (changelog != null) {
@Override public Map<String, String> toConfig(Config jobConfig) { Map<String, String> tableConfig = new HashMap<>(super.toConfig(jobConfig)); // Store factory configuration tableConfig.put(String.format(StorageConfig.FACTORY(), tableId), InMemoryKeyValueStorageEngineFactory.class.getName()); return Collections.unmodifiableMap(tableConfig); } }
taskModel.getChangelogPartition(), systemAdmins, new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(), new SystemClock());
@Test public void testChangelogEnabledWithCustomParameters() { Map<String, String> tableConfig = createTableDescriptor() .withChangelogStream("my-stream") .withChangelogReplicationFactor(100) .toConfig(createJobConfig()); Assert.assertEquals(3, tableConfig.size()); Assert.assertEquals("my-stream", String.format( tableConfig.get(String.format(StorageConfig.CHANGELOG_STREAM(), TABLE_ID)))); Assert.assertEquals("100", String.format( tableConfig.get(String.format(StorageConfig.CHANGELOG_REPLICATION_FACTOR(), TABLE_ID)))); }
/** * Builds list of {@link Task} from job model in coordinator stream. * @param consumer system consumer associated with a job's coordinator stream. * @return list of {@link Task} constructed from job model in coordinator stream. */ protected List<Task> readTasksFromCoordinatorStream(CoordinatorStreamSystemConsumer consumer) { LocalityManager localityManager = new LocalityManager(consumer.getConfig(), new MetricsRegistryMap()); Map<String, Map<String, String>> containerIdToHostMapping = localityManager.readContainerLocality(); TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(consumer.getConfig(), new MetricsRegistryMap()); Map<String, String> taskNameToContainerIdMapping = taskAssignmentManager.readTaskAssignment(); StorageConfig storageConfig = new StorageConfig(consumer.getConfig()); List<String> storeNames = JavaConverters.seqAsJavaListConverter(storageConfig.getStoreNames()).asJava(); return taskNameToContainerIdMapping.entrySet() .stream() .map(entry -> { String hostName = containerIdToHostMapping.get(entry.getValue()).get(SetContainerHostMapping.HOST_KEY); return new Task(hostName, entry.getKey(), entry.getValue(), new ArrayList<>(), storeNames); }).collect(Collectors.toList()); } }
@Test public void testChangelogDisabled() { Map<String, String> tableConfig = createTableDescriptor() .toConfig(createJobConfig()); Assert.assertEquals(1, tableConfig.size()); Assert.assertFalse(tableConfig.containsKey(String.format(StorageConfig.CHANGELOG_STREAM(), TABLE_ID))); }
StorageConfig storageConfig = new StorageConfig(prevConfig); for (String store : JavaConversions.asJavaCollection(storageConfig.getStoreNames())) { String changelog = storageConfig.getChangelogStream(store) .getOrElse(defaultValue(null)); if (changelog != null) {
@Override public Map<String, String> generateConfig(Config jobConfig, Map<String, String> generatedConfig) { Map<String, String> tableConfig = new HashMap<>(); // Store factory configuration tableConfig.put(String.format( StorageConfig.FACTORY(), tableSpec.getId()), InMemoryKeyValueStorageEngineFactory.class.getName()); // Common store configuration tableConfig.putAll(generateCommonStoreConfig(jobConfig, generatedConfig)); // Rest of the configuration tableSpec.getConfig().entrySet().stream() .filter(e -> !e.getKey().startsWith("internal.")) .forEach(e -> { String k = e.getKey(); String v = e.getValue(); String realKey = k.startsWith("inmemory.") ? String.format("stores.%s", tableSpec.getId()) + "." + k.substring("inmemory.".length()) : String.format(JavaTableConfig.TABLE_ID_PREFIX, tableSpec.getId()) + "." + k; tableConfig.put(realKey, v); }); logger.info("Generated configuration for table " + tableSpec.getId()); return tableConfig; }
taskModel.getChangelogPartition(), systemAdmins, new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(), new SystemClock());
assertEquals("snappy", RocksDbTableDescriptor.ROCKSDB_COMPRESSION, tableConfig); assertEquals("fifo", RocksDbTableDescriptor.ROCKSDB_COMPACTION_STYLE, tableConfig); Assert.assertFalse(tableConfig.containsKey(String.format(StorageConfig.CHANGELOG_STREAM(), TABLE_ID))); Assert.assertFalse(tableConfig.containsKey(String.format(StorageConfig.CHANGELOG_REPLICATION_FACTOR(), TABLE_ID))); Assert.assertEquals("xyz", tableConfig.get("abc"));
@Test public void testChangelogEnabled() { Map<String, String> tableConfig = createTableDescriptor() .withChangelogEnabled() .toConfig(createJobConfig()); Assert.assertEquals(2, tableConfig.size()); Assert.assertEquals("test-job-10-table-t1", String.format( tableConfig.get(String.format(StorageConfig.CHANGELOG_STREAM(), TABLE_ID)))); }
public JavaStorageConfig getStorageConfigs() { HashMap<String, String> configs = new HashMap<>(); configs.put(String.format(StorageConfig.FACTORY(), this.getStoreName()), this.getStoreFactory()); configs.put(String.format(StorageConfig.CHANGELOG_STREAM(), this.getStoreName()), this.getChangelogStream()); configs.putAll(this.getOtherProperties()); return new JavaStorageConfig(new MapConfig(configs)); }
storeConfig.put(String.format(StorageConfig.KEY_SERDE(), tableSpec.getId()), keySerde); storeConfig.put(String.format(StorageConfig.MSG_SERDE(), tableSpec.getId()), valueSerde); storeConfig.put(String.format(StorageConfig.CHANGELOG_STREAM(), tableSpec.getId()), changelogStream); storeConfig.put(String.format(StorageConfig.CHANGELOG_REPLICATION_FACTOR(), tableSpec.getId()), changelogReplicationFactor);