@Override public KV<Object, Object> apply(SamzaSqlRelMessage message) { return this.samzaMsgConverter.convertToSamzaMessage(message); } }
@Override public RelDataType toSql(RelDataType type) { return convertToSql(this, type); }
@Override public SamzaSqlRelMessage apply(KV<Object, Object> message) { return this.msgConverter.convertToRelMessage(message); } }
.collect(Collectors.toMap(Function.identity(), x -> ioResolver.fetchSinkInfo(x))); .collect(Collectors.toMap(Function.identity(), src -> ioResolver.fetchSourceInfo(src))); x -> initializePlugin("RelSchemaProvider", x.getRelSchemaProviderName(), staticConfig, CFG_FMT_REL_SCHEMA_PROVIDER_DOMAIN, (o, c) -> ((RelSchemaProviderFactory) o).create(x.getSystemStream(), c)))); x -> initializePlugin("SamzaRelConverter", x.getSamzaRelConverterName(), staticConfig, CFG_FMT_SAMZA_REL_CONVERTER_DOMAIN, (o, c) -> ((SamzaRelConverterFactory) o).create(x.getSystemStream(), relSchemaProvidersBySource.get(x.getSource()), c)))); x -> initializePlugin("SamzaRelTableKeyConverter", x.getSamzaRelTableKeyConverterName(), staticConfig, CFG_FMT_SAMZA_REL_TABLE_KEY_CONVERTER_DOMAIN, (o, c) -> ((SamzaRelTableKeyConverterFactory) o).create(x.getSystemStream(), c)))); udfMetadata = udfResolver.getUdfs();
@Override public SqlSchema getTableSchema(ExecutionContext context, String tableName) { /** * currently Shell works only for systems that has Avro schemas */ lastErrorMsg = ""; int execId = execIdSeq.incrementAndGet(); Map<String, String> staticConfigs = fetchSamzaSqlConfig(execId, context); Config samzaSqlConfig = new MapConfig(staticConfigs); SqlSchema sqlSchema = null; try { SqlIOResolver ioResolver = SamzaSqlApplicationConfig.createIOResolver(samzaSqlConfig); SqlIOConfig sourceInfo = ioResolver.fetchSourceInfo(tableName); RelSchemaProvider schemaProvider = SamzaSqlApplicationConfig.initializePlugin("RelSchemaProvider", sourceInfo.getRelSchemaProviderName(), samzaSqlConfig, SamzaSqlApplicationConfig.CFG_FMT_REL_SCHEMA_PROVIDER_DOMAIN, (o, c) -> ((RelSchemaProviderFactory) o).create(sourceInfo.getSystemStream(), c)); sqlSchema = schemaProvider.getSqlSchema(); } catch (SamzaException ex) { String msg = "getTableSchema failed with exception "; lastErrorMsg = msg + ex.toString(); LOG.error(msg, ex); } return sqlSchema; }
@Override public Connection connect(String url, Properties info) throws SQLException { if (!acceptsURL(url)) { return null; } final String prefix = getConnectStringPrefix(); assert url.startsWith(prefix); final String urlSuffix = url.substring(prefix.length()); final Properties info2 = ConnectStringParser.parse(urlSuffix, info); final AvaticaConnection connection = ((CalciteFactory) factory).newConnection(this, factory, url, info2, null, typeFactory); handler.onConnectionInit(connection); return connection; } }
private JoinInputNode.InputType getInputType(RelNode relNode, TranslatorContext context) { // NOTE: Any intermediate form of a join is always a stream. Eg: For the second level join of // stream-table-table join, the left side of the join is join output, which we always // assume to be a stream. The intermediate stream won't be an instance of EnumerableTableScan. // The join key(s) for the table could be an udf in which case the relNode would be LogicalProject. if (relNode instanceof EnumerableTableScan || relNode instanceof LogicalProject) { SqlIOConfig sourceTableConfig = resolveSourceConfigForTable(relNode, context); if (sourceTableConfig == null || !sourceTableConfig.getTableDescriptor().isPresent()) { return JoinInputNode.InputType.STREAM; } else if (sourceTableConfig.getTableDescriptor().get() instanceof RemoteTableDescriptor || sourceTableConfig.getTableDescriptor().get() instanceof CachingTableDescriptor) { return JoinInputNode.InputType.REMOTE_TABLE; } else { return JoinInputNode.InputType.LOCAL_TABLE; } } else { return JoinInputNode.InputType.STREAM; } }
private SqlIOConfig resolveSourceConfig(RelNode relNode) { String sourceName = String.join(".", relNode.getTable().getQualifiedName()); SqlIOConfig sourceConfig = ioResolver.fetchSourceInfo(sourceName); if (sourceConfig == null) { throw new SamzaException("Unsupported source found in join statement: " + sourceName); } return sourceConfig; }
String getSourceName() { return SqlIOConfig.getSourceFromSourceParts(relNode.getTable().getQualifiedName()); }
public ScalarUdf createInstance(String clazz, String udfName) { Config udfConfig = udfMetadata.get(udfName).getUdfConfig(); ScalarUdf scalarUdf = ReflectionUtils.createInstance(clazz); if (scalarUdf == null) { String msg = String.format("Couldn't create udf %s of class %s", udfName, clazz); throw new SamzaException(msg); } scalarUdf.init(udfConfig); return scalarUdf; }
public static SqlIOResolver createIOResolver(Config config) { String sourceResolveValue = config.get(CFG_IO_RESOLVER); Validate.notEmpty(sourceResolveValue, "ioResolver config is not set or empty"); return initializePlugin("SqlIOResolver", sourceResolveValue, config, CFG_FMT_SOURCE_RESOLVER_DOMAIN, (o, c) -> ((SqlIOResolverFactory) o).create(c, config)); }
@Override public boolean apply(SamzaSqlInputMessage samzaSqlInputMessage) { return !relConverter.isSystemMessage(samzaSqlInputMessage.getKeyAndMessageKV()); } }
@Override public Object getMessageKey(SamzaSqlRelMessage message) { SamzaSqlRelRecord keyRecord = getMessageKeyRelRecord(message); // If all the message key rel record values are null, return null message key. if (keyRecord.getFieldValues().stream().allMatch(Objects::isNull)) { return null; } // Using the table key converter, convert message key from rel format to the record key format. return relTableKeyConverter.convertToTableKeyFormat(keyRecord); }
public SamzaSqlApplicationConfig(Config staticConfig, Set<String> inputSystemStreams, Set<String> outputSystemStreams) { ioResolver = createIOResolver(staticConfig); inputSystemStreamConfigBySource = inputSystemStreams.stream() .collect(Collectors.toMap(Function.identity(), src -> ioResolver.fetchSourceInfo(src))); outputSystemStreamConfigsBySource = outputSystemStreams.stream() .collect(Collectors.toMap(Function.identity(), x -> ioResolver.fetchSinkInfo(x))); systemStreamConfigsBySource = new HashMap<>(inputSystemStreamConfigBySource); systemStreamConfigsBySource.putAll(outputSystemStreamConfigsBySource); Set<SqlIOConfig> systemStreamConfigs = new HashSet<>(systemStreamConfigsBySource.values()); relSchemaProvidersBySource = systemStreamConfigs.stream() .collect(Collectors.toMap(SqlIOConfig::getSource, x -> initializePlugin("RelSchemaProvider", x.getRelSchemaProviderName(), staticConfig, CFG_FMT_REL_SCHEMA_PROVIDER_DOMAIN, (o, c) -> ((RelSchemaProviderFactory) o).create(x.getSystemStream(), c)))); samzaRelConvertersBySource = systemStreamConfigs.stream() .collect(Collectors.toMap(SqlIOConfig::getSource, x -> initializePlugin("SamzaRelConverter", x.getSamzaRelConverterName(), staticConfig, CFG_FMT_SAMZA_REL_CONVERTER_DOMAIN, (o, c) -> ((SamzaRelConverterFactory) o).create(x.getSystemStream(), relSchemaProvidersBySource.get(x.getSource()), c)))); udfResolver = createUdfResolver(staticConfig); udfMetadata = udfResolver.getUdfs(); windowDurationMs = staticConfig.getLong(CFG_GROUPBY_WINDOW_DURATION_MS, DEFAULT_GROUPBY_WINDOW_DURATION_MS); // remove the SqlIOConfigs of outputs whose system is "log" out of systemStreamConfigsBySource outputSystemStreamConfigsBySource.forEach((k, v) -> { if (k.split("\\.")[0].equals(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG)) { systemStreamConfigsBySource.remove(k); } }); }
private boolean isTable(RelNode relNode) { // NOTE: Any intermediate form of a join is always a stream. Eg: For the second level join of // stream-table-table join, the left side of the join is join output, which we always // assume to be a stream. The intermediate stream won't be an instance of EnumerableTableScan. if (relNode instanceof EnumerableTableScan) { return resolveSourceConfig(relNode).getTableDescriptor().isPresent(); } else { return false; } }
@Override public KV<Object, Object> apply(SamzaSqlRelMessage message) { return this.samzaMsgConverter.convertToSamzaMessage(message); } }
@Override protected List<Object> getTableRelRecordFieldValues(KV record) { // Using the message rel converter, convert message to sql rel message and add to output values. SamzaSqlRelMessage relMessage = msgConverter.convertToRelMessage(record); return relMessage.getSamzaSqlRelRecord().getFieldValues(); }
@Override public RelDataType toSql(RelDataType type) { return convertToSql(this, type); }
@Override public KV<Object, Object> apply(SamzaSqlRelMessage message) { Instant beginProcessing = Instant.now(); KV<Object, Object> retKV = this.samzaMsgConverter.convertToSamzaMessage(message); updateMetrics(beginProcessing, Instant.now(), message.getSamzaSqlRelMsgMetadata()); return retKV; }
@Override public SamzaSqlRelMessage apply(SamzaSqlInputMessage samzaSqlInputMessage) { Instant startProcessing = Instant.now(); /* SAMZA-2089/LISAMZA-10654: the SamzaRelConverter.convertToRelMessage currently does not initialize * the samzaSqlRelMessage.samzaSqlRelMsgMetadata, this needs to be fixed */ SamzaSqlRelMessage retMsg = this.msgConverter.convertToRelMessage(samzaSqlInputMessage.getKeyAndMessageKV()); retMsg.setEventTime(samzaSqlInputMessage.getMetadata().getEventTime()); retMsg.setArrivalTime(samzaSqlInputMessage.getMetadata().getarrivalTime()); retMsg.setScanTime(startProcessing.toString()); updateMetrics(startProcessing, Instant.now()); return retMsg; }