public static SqlIOResolver createIOResolver(Config config) { String sourceResolveValue = config.get(CFG_IO_RESOLVER); Map<String, String> metadataPrefixProperties = new HashMap<>(); metadataPrefixProperties.put( String.format(CFG_FMT_SOURCE_RESOLVER_DOMAIN, sourceResolveValue) + CFG_METADATA_TOPIC_PREFIX, config.get(CFG_METADATA_TOPIC_PREFIX, DEFAULT_METADATA_TOPIC_PREFIX)); Config newConfig = new MapConfig(Arrays.asList(config, metadataPrefixProperties)); Validate.notEmpty(sourceResolveValue, "ioResolver config is not set or empty"); return initializePlugin("SqlIOResolver", sourceResolveValue, newConfig, CFG_FMT_SOURCE_RESOLVER_DOMAIN, (o, c) -> ((SqlIOResolverFactory) o).create(c, newConfig)); }
public static SqlIOResolver createIOResolver(Config config) { String sourceResolveValue = config.get(CFG_IO_RESOLVER); Validate.notEmpty(sourceResolveValue, "ioResolver config is not set or empty"); return initializePlugin("SqlIOResolver", sourceResolveValue, config, CFG_FMT_SOURCE_RESOLVER_DOMAIN, (o, c) -> ((SqlIOResolverFactory) o).create(c, config)); }
x -> initializePlugin("RelSchemaProvider", x.getRelSchemaProviderName(), staticConfig, CFG_FMT_REL_SCHEMA_PROVIDER_DOMAIN, (o, c) -> ((RelSchemaProviderFactory) o).create(x.getSystemStream(), c)))); x -> initializePlugin("SamzaRelConverter", x.getSamzaRelConverterName(), staticConfig, CFG_FMT_SAMZA_REL_CONVERTER_DOMAIN, (o, c) -> ((SamzaRelConverterFactory) o).create(x.getSystemStream(), relSchemaProvidersBySource.get(x.getSource()), c)))); .filter(SqlIOConfig::isRemoteTable) .collect(Collectors.toMap(SqlIOConfig::getSource, x -> initializePlugin("SamzaRelTableKeyConverter", x.getSamzaRelTableKeyConverterName(), staticConfig, CFG_FMT_SAMZA_REL_TABLE_KEY_CONVERTER_DOMAIN, (o, c) -> ((SamzaRelTableKeyConverterFactory) o).create(x.getSystemStream(), c))));
public SamzaSqlApplicationConfig(Config staticConfig, Set<String> inputSystemStreams, Set<String> outputSystemStreams) { ioResolver = createIOResolver(staticConfig); inputSystemStreamConfigBySource = inputSystemStreams.stream() .collect(Collectors.toMap(Function.identity(), src -> ioResolver.fetchSourceInfo(src))); outputSystemStreamConfigsBySource = outputSystemStreams.stream() .collect(Collectors.toMap(Function.identity(), x -> ioResolver.fetchSinkInfo(x))); systemStreamConfigsBySource = new HashMap<>(inputSystemStreamConfigBySource); systemStreamConfigsBySource.putAll(outputSystemStreamConfigsBySource); Set<SqlIOConfig> systemStreamConfigs = new HashSet<>(systemStreamConfigsBySource.values()); relSchemaProvidersBySource = systemStreamConfigs.stream() .collect(Collectors.toMap(SqlIOConfig::getSource, x -> initializePlugin("RelSchemaProvider", x.getRelSchemaProviderName(), staticConfig, CFG_FMT_REL_SCHEMA_PROVIDER_DOMAIN, (o, c) -> ((RelSchemaProviderFactory) o).create(x.getSystemStream(), c)))); samzaRelConvertersBySource = systemStreamConfigs.stream() .collect(Collectors.toMap(SqlIOConfig::getSource, x -> initializePlugin("SamzaRelConverter", x.getSamzaRelConverterName(), staticConfig, CFG_FMT_SAMZA_REL_CONVERTER_DOMAIN, (o, c) -> ((SamzaRelConverterFactory) o).create(x.getSystemStream(), relSchemaProvidersBySource.get(x.getSource()), c)))); udfResolver = createUdfResolver(staticConfig); udfMetadata = udfResolver.getUdfs(); windowDurationMs = staticConfig.getLong(CFG_GROUPBY_WINDOW_DURATION_MS, DEFAULT_GROUPBY_WINDOW_DURATION_MS); // remove the SqlIOConfigs of outputs whose system is "log" out of systemStreamConfigsBySource outputSystemStreamConfigsBySource.forEach((k, v) -> { if (k.split("\\.")[0].equals(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG)) { systemStreamConfigsBySource.remove(k); } }); }
@Override public SqlSchema getTableSchema(ExecutionContext context, String tableName) { /** * currently Shell works only for systems that has Avro schemas */ lastErrorMsg = ""; int execId = execIdSeq.incrementAndGet(); Map<String, String> staticConfigs = fetchSamzaSqlConfig(execId, context); Config samzaSqlConfig = new MapConfig(staticConfigs); SqlSchema sqlSchema = null; try { SqlIOResolver ioResolver = SamzaSqlApplicationConfig.createIOResolver(samzaSqlConfig); SqlIOConfig sourceInfo = ioResolver.fetchSourceInfo(tableName); RelSchemaProvider schemaProvider = SamzaSqlApplicationConfig.initializePlugin("RelSchemaProvider", sourceInfo.getRelSchemaProviderName(), samzaSqlConfig, SamzaSqlApplicationConfig.CFG_FMT_REL_SCHEMA_PROVIDER_DOMAIN, (o, c) -> ((RelSchemaProviderFactory) o).create(sourceInfo.getSystemStream(), c)); sqlSchema = schemaProvider.getSqlSchema(); } catch (SamzaException ex) { String msg = "getTableSchema failed with exception "; lastErrorMsg = msg + ex.toString(); LOG.error(msg, ex); } return sqlSchema; }