How to use

getOutputSystemStreamConfigsBySource

method

in

org.apache.samza.sql.runner.SamzaSqlApplicationConfig

@Test
public void testConfigInit() {
 Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10);
 config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, "Insert into testavro.COMPLEX1 select * from testavro.SIMPLE1");
 String configUdfResolverDomain = String.format(SamzaSqlApplicationConfig.CFG_FMT_UDF_RESOLVER_DOMAIN, "config");
 int numUdfs = config.get(configUdfResolverDomain + ConfigBasedUdfResolver.CFG_UDF_CLASSES).split(",").length;
 List<String> sqlStmts = fetchSqlFromConfig(config);
 List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
 SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream)
     .collect(Collectors.toList()),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
 Assert.assertEquals(numUdfs, samzaSqlApplicationConfig.getUdfMetadata().size());
 Assert.assertEquals(1, samzaSqlApplicationConfig.getInputSystemStreamConfigBySource().size());
 Assert.assertEquals(1, samzaSqlApplicationConfig.getOutputSystemStreamConfigsBySource().size());
}

  new ScanTranslator(sqlConfig.getSamzaRelConverters(), sqlConfig.getInputSystemStreamConfigBySource(), queryId);
ModifyTranslator modifyTranslator =
  new ModifyTranslator(sqlConfig.getSamzaRelConverters(), sqlConfig.getOutputSystemStreamConfigsBySource(), queryId);
sqlConfig.getOutputSystemStreamConfigsBySource().keySet().forEach(
  key -> {
   if (key.split("\\.")[0].equals(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG)) {

@Test
public void testGetInputAndOutputStreamConfigsFanOut() {
 List<String> sqlStmts = Arrays.asList("Insert into testavro.COMPLEX1 select * from testavro.SIMPLE1",
   "insert into testavro.Profile select * from testavro.SIMPLE1");
 Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10);
 config.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
 List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
 SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream)
     .collect(Collectors.toList()),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
 Set<String> inputKeys = samzaSqlApplicationConfig.getInputSystemStreamConfigBySource().keySet();
 Set<String> outputKeys = samzaSqlApplicationConfig.getOutputSystemStreamConfigsBySource().keySet();
 List<String> outputStreamList = samzaSqlApplicationConfig.getOutputSystemStreams();
 Assert.assertEquals(1, inputKeys.size());
 Assert.assertTrue(inputKeys.contains("testavro.SIMPLE1"));
 Assert.assertEquals(2, outputKeys.size());
 Assert.assertTrue(outputKeys.contains("testavro.COMPLEX1"));
 Assert.assertTrue(outputKeys.contains("testavro.Profile"));
 Assert.assertEquals(2, outputStreamList.size());
 Assert.assertEquals("testavro.COMPLEX1", outputStreamList.get(0));
 Assert.assertEquals("testavro.Profile", outputStreamList.get(1));
}

@Test
public void testGetInputAndOutputStreamConfigsFanIn() {
 List<String> sqlStmts = Arrays.asList("Insert into testavro.COMPLEX1 select * from testavro.SIMPLE1",
   "insert into testavro.COMPLEX1 select * from testavro.SIMPLE2");
 Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10);
 config.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
 List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
 SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream)
     .collect(Collectors.toList()),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
 Set<String> inputKeys = samzaSqlApplicationConfig.getInputSystemStreamConfigBySource().keySet();
 Set<String> outputKeys = samzaSqlApplicationConfig.getOutputSystemStreamConfigsBySource().keySet();
 List<String> outputStreamList = samzaSqlApplicationConfig.getOutputSystemStreams();
 Assert.assertEquals(2, inputKeys.size());
 Assert.assertTrue(inputKeys.contains("testavro.SIMPLE1"));
 Assert.assertTrue(inputKeys.contains("testavro.SIMPLE2"));
 Assert.assertEquals(1, outputKeys.size());
 Assert.assertTrue(outputKeys.contains("testavro.COMPLEX1"));
 Assert.assertEquals(2, outputStreamList.size());
 Assert.assertEquals("testavro.COMPLEX1", outputStreamList.get(0));
 Assert.assertEquals("testavro.COMPLEX1", outputStreamList.get(1));
}

 private void sendToOutputStream(String queryLogicalId, String logicalOpId, String sinkStream, StreamApplicationDescriptor appDesc, TranslatorContext translatorContext, RelNode node, int queryId) {
  SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(sinkStream);
  MessageStream<SamzaSqlRelMessage> stream = translatorContext.getMessageStream(node.getId());
  MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(queryLogicalId, logicalOpId, sinkStream, queryId));
  Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor();
  if (!tableDescriptor.isPresent()) {
   KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>());
   String systemName = sinkConfig.getSystemName();
   DelegatingSystemDescriptor
     sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new);
   GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamId(), noOpKVSerde);
   OutputStream stm = outputMsgStreams.computeIfAbsent(sinkConfig.getSource(), v -> appDesc.getOutputStream(osd));
   outputStream.sendTo(stm);
  } else {
   Table outputTable = appDesc.getTable(tableDescriptor.get());
   if (outputTable == null) {
    String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource();
    throw new SamzaException(msg);
   }
   outputStream.sendTo(outputTable);
  }
 }
}

 private void sendToOutputStream(StreamApplicationDescriptor appDesc, TranslatorContext context, RelNode node, int queryId) {
  SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG);
  MessageStream<SamzaSqlRelMessage> stream = context.getMessageStream(node.getId());
  MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG, queryId));
  Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor();
  if (!tableDescriptor.isPresent()) {
   KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>());
   String systemName = sinkConfig.getSystemName();
   DelegatingSystemDescriptor
     sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new);
   GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamName(), noOpKVSerde);
   if (OutputMapFunction.logOutputStream == null) {
    OutputMapFunction.logOutputStream = appDesc.getOutputStream(osd);
   }
   outputStream.sendTo(OutputMapFunction.logOutputStream);
  } else {
   Table outputTable = appDesc.getTable(tableDescriptor.get());
   if (outputTable == null) {
    String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource();
    throw new SamzaException(msg);
   }
   outputStream.sendTo(outputTable);
  }
 }
}