private Config fetchSystemConfigs(String systemName) { return config.subset(systemName + "."); } }
public Config subset(String prefix) { return subset(prefix, true); }
private Config fetchSystemConfigs(String systemName) { return config.subset(systemName + "."); } }
public Config subset(String prefix) { return subset(prefix, true); }
/** * Get the config subset for fs.<scheme>.impl * It can include config for fs.<scheme>.impl and additional config for the subKeys fs.<scheme>.impl.* from the configuration * e.g. for scheme "myScheme", there could be config for fs.myScheme.impl, fs.myScheme.impl.client and fs.myScheme.impl.server * @param scheme scheme name, such as http, hdfs, myscheme * @return config for the particular scheme */ public Config getSchemeConfig(final String scheme) { String fsSchemeImpl = String.format(FS_IMPL_TEMPLATE, scheme); Config schemeConfig = config.subset(fsSchemeImpl, false); // do not strip off the prefix return schemeConfig; } }
/** * Get the config subset for fs.<scheme>.impl * It can include config for fs.<scheme>.impl and additional config for the subKeys fs.<scheme>.impl.* from the configuration * e.g. for scheme "myScheme", there could be config for fs.myScheme.impl, fs.myScheme.impl.client and fs.myScheme.impl.server * @param scheme scheme name, such as http, hdfs, myscheme * @return config for the particular scheme */ public Config getSchemeConfig(final String scheme) { String fsSchemeImpl = String.format(FS_IMPL_TEMPLATE, scheme); Config schemeConfig = config.subset(fsSchemeImpl, false); // do not strip off the prefix return schemeConfig; } }
public List<String> getResourceNames() { Config subConfig = config.subset(RESOURCE_PREFIX, true); List<String> resourceNames = new ArrayList<String>(); for (String key : subConfig.keySet()) { if (key.endsWith(PATH_SUFFIX)) { resourceNames.add(key.substring(0, key.length() - PATH_SUFFIX.length())); } } return resourceNames; }
public List<String> getResourceNames() { Config subConfig = config.subset(RESOURCE_PREFIX, true); List<String> resourceNames = new ArrayList<String>(); for (String key : subConfig.keySet()) { if (key.endsWith(PATH_SUFFIX)) { resourceNames.add(key.substring(0, key.length() - PATH_SUFFIX.length())); } } return resourceNames; }
public ElasticsearchConfig(String name, Config config) { super(config.subset("systems." + name + ".")); logAllSettings(this); }
/** * Get all schemes * @return List of schemes in strings */ public List<String> getSchemes() { Config subConfig = config.subset(FS_IMPL_PREFIX, true); List<String> schemes = new ArrayList<String>(); for (String key : subConfig.keySet()) { if (key.endsWith(FS_IMPL_SUFFIX)) { schemes.add(key.substring(0, key.length() - FS_IMPL_SUFFIX.length())); } } return schemes; }
/** * Get all schemes * @return List of schemes in strings */ public List<String> getSchemes() { Config subConfig = config.subset(FS_IMPL_PREFIX, true); List<String> schemes = new ArrayList<String>(); for (String key : subConfig.keySet()) { if (key.endsWith(FS_IMPL_SUFFIX)) { schemes.add(key.substring(0, key.length() - FS_IMPL_SUFFIX.length())); } } return schemes; }
/** * * Groups configuration defined in the config object for each of the monitors into a MonitorConfig object * @param config contains the entire configuration defined for all the monitors. * @return a map of monitorName, {@link MonitorConfig}, where each MonitorConfig object * contains all the configuration defined for the monitor named monitorName. */ public static Map<String, MonitorConfig> getMonitorConfigs(Config config) { Map<String, MonitorConfig> monitorConfigMap = new HashMap<>(); Config monitorConfig = config.subset(MONITOR_PREFIX); for (String monitorName : getMonitorNames(monitorConfig)) { monitorConfigMap.put(monitorName, new MonitorConfig(monitorConfig.subset(monitorName + MONITOR_CONFIG_KEY_SEPARATOR))); } return monitorConfigMap; }
public static <T> T initializePlugin(String pluginName, String plugin, Config staticConfig, String pluginDomainFormat, BiFunction<Object, Config, T> factoryInvoker) { String pluginDomain = String.format(pluginDomainFormat, plugin); Config pluginConfig = staticConfig.subset(pluginDomain); String factoryName = pluginConfig.getOrDefault(CFG_FACTORY, ""); Validate.notEmpty(factoryName, String.format("Factory is not set for %s", plugin)); Object factory = ReflectionUtils.createInstance(factoryName); Validate.notNull(factory, String.format("Factory creation failed for %s", plugin)); LOG.info("Instantiating {} using factory {} with props {}", pluginName, factoryName, pluginConfig); return factoryInvoker.apply(factory, pluginConfig); }
public static <T> T initializePlugin(String pluginName, String plugin, Config staticConfig, String pluginDomainFormat, BiFunction<Object, Config, T> factoryInvoker) { String pluginDomain = String.format(pluginDomainFormat, plugin); Config pluginConfig = staticConfig.subset(pluginDomain); String factoryName = pluginConfig.getOrDefault(CFG_FACTORY, ""); Validate.notEmpty(factoryName, String.format("Factory is not set for %s", plugin)); Object factory = ReflectionUtils.createInstance(factoryName); Validate.notNull(factory, String.format("Factory creation failed for %s", plugin)); LOG.info("Instantiating {} using factory {} with props {}", pluginName, factoryName, pluginConfig); return factoryInvoker.apply(factory, pluginConfig); }
@Override public Config rewrite(String name, Config config) { Map<String, String> configMap = new HashMap<>(config); configMap.putAll(config.subset(String.format("job.config.rewriter.%s.", name))); return new MapConfig(configMap); } }
private void validateStreamSerdeConfigure(String streamId, Config config, Map<String, Serde> deserializedSerdes) { Config streamConfig = config.subset(String.format("streams.%s.samza.", streamId)); String keySerdeName = streamConfig.get("key.serde"); String valueSerdeName = streamConfig.get("msg.serde"); assertTrue(String.format("Serialized serdes should contain %s key serde", streamId), deserializedSerdes.containsKey(keySerdeName)); assertTrue(String.format("Serialized %s key serde should be a StringSerde", streamId), keySerdeName.startsWith(StringSerde.class.getSimpleName())); assertTrue(String.format("Serialized serdes should contain %s msg serde", streamId), deserializedSerdes.containsKey(valueSerdeName)); assertTrue(String.format("Serialized %s msg serde should be a JsonSerdeV2", streamId), valueSerdeName.startsWith(JsonSerdeV2.class.getSimpleName())); }
/** * Construct a table manager instance * @param config job configuration * @param serdes Serde instances for tables */ public TableManager(Config config, Map<String, Serde<Object>> serdes) { new JavaTableConfig(config).getTableIds().forEach(tableId -> { // Construct the table provider String tableProviderFactory = config.get(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, tableId)); // Construct the KVSerde JavaTableConfig tableConfig = new JavaTableConfig(config); KVSerde serde = KVSerde.of( serdes.get(tableConfig.getKeySerde(tableId)), serdes.get(tableConfig.getValueSerde(tableId))); TableSpec tableSpec = new TableSpec(tableId, serde, tableProviderFactory, config.subset(String.format(JavaTableConfig.TABLE_ID_PREFIX, tableId) + ".")); addTable(tableSpec); logger.info("Added table " + tableSpec.getId()); }); }
private void validateTableSerdeConfigure(String tableId, Config config, Map<String, Serde> deserializedSerdes) { Config streamConfig = config.subset(String.format("stores.%s.", tableId)); String keySerdeName = streamConfig.get("key.serde"); String valueSerdeName = streamConfig.get("msg.serde"); assertTrue(String.format("Serialized serdes should contain %s key serde", tableId), deserializedSerdes.containsKey(keySerdeName)); assertTrue(String.format("Serialized %s key serde should be a StringSerde", tableId), keySerdeName.startsWith(StringSerde.class.getSimpleName())); assertTrue(String.format("Serialized serdes should contain %s value serde", tableId), deserializedSerdes.containsKey(valueSerdeName)); assertTrue(String.format("Serialized %s msg serde should be a JsonSerdeV2", tableId), valueSerdeName.startsWith(JsonSerdeV2.class.getSimpleName())); }
private Map<String, Serde> validateAndGetDeserializedSerdes(Config jobConfig, int numSerdes) { Config serializers = jobConfig.subset("serializers.registry.", true); // make sure that the serializers deserialize correctly SerializableSerde<Serde> serializableSerde = new SerializableSerde<>(); assertEquals(numSerdes, serializers.size()); return serializers.entrySet().stream().collect(Collectors.toMap( e -> e.getKey().replace(SerializerConfig.SERIALIZED_INSTANCE_SUFFIX(), ""), e -> serializableSerde.fromBytes(Base64.getDecoder().decode(e.getValue().getBytes())) )); }
private void validateIntermediateStreamConfigure(String streamId, String physicalName, Config config) { Config intStreamConfig = config.subset(String.format("streams.%s.", streamId), true); assertEquals("intermediate-system", intStreamConfig.get("samza.system")); assertEquals(String.valueOf(Integer.MAX_VALUE), intStreamConfig.get("samza.priority")); assertEquals("true", intStreamConfig.get("samza.delete.committed.messages")); assertEquals(physicalName, intStreamConfig.get("samza.physical.name")); assertEquals("true", intStreamConfig.get("samza.intermediate")); assertEquals("oldest", intStreamConfig.get("samza.offset.default")); }