Tabnine Logo
SamzaSqlApplicationConfig.getOutputSystemStreamConfigsBySource
Code IndexAdd Tabnine to your IDE (free)

How to use
getOutputSystemStreamConfigsBySource
method
in
org.apache.samza.sql.runner.SamzaSqlApplicationConfig

Best Java code snippets using org.apache.samza.sql.runner.SamzaSqlApplicationConfig.getOutputSystemStreamConfigsBySource (Showing top 6 results out of 315)

origin: apache/samza

@Test
public void testConfigInit() {
 Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10);
 config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, "Insert into testavro.COMPLEX1 select * from testavro.SIMPLE1");
 String configUdfResolverDomain = String.format(SamzaSqlApplicationConfig.CFG_FMT_UDF_RESOLVER_DOMAIN, "config");
 int numUdfs = config.get(configUdfResolverDomain + ConfigBasedUdfResolver.CFG_UDF_CLASSES).split(",").length;
 List<String> sqlStmts = fetchSqlFromConfig(config);
 List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
 SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream)
     .collect(Collectors.toList()),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
 Assert.assertEquals(numUdfs, samzaSqlApplicationConfig.getUdfMetadata().size());
 Assert.assertEquals(1, samzaSqlApplicationConfig.getInputSystemStreamConfigBySource().size());
 Assert.assertEquals(1, samzaSqlApplicationConfig.getOutputSystemStreamConfigsBySource().size());
}
origin: org.apache.samza/samza-sql

  new ScanTranslator(sqlConfig.getSamzaRelConverters(), sqlConfig.getInputSystemStreamConfigBySource(), queryId);
ModifyTranslator modifyTranslator =
  new ModifyTranslator(sqlConfig.getSamzaRelConverters(), sqlConfig.getOutputSystemStreamConfigsBySource(), queryId);
sqlConfig.getOutputSystemStreamConfigsBySource().keySet().forEach(
  key -> {
   if (key.split("\\.")[0].equals(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG)) {
origin: apache/samza

@Test
public void testGetInputAndOutputStreamConfigsFanOut() {
 List<String> sqlStmts = Arrays.asList("Insert into testavro.COMPLEX1 select * from testavro.SIMPLE1",
   "insert into testavro.Profile select * from testavro.SIMPLE1");
 Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10);
 config.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
 List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
 SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream)
     .collect(Collectors.toList()),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
 Set<String> inputKeys = samzaSqlApplicationConfig.getInputSystemStreamConfigBySource().keySet();
 Set<String> outputKeys = samzaSqlApplicationConfig.getOutputSystemStreamConfigsBySource().keySet();
 List<String> outputStreamList = samzaSqlApplicationConfig.getOutputSystemStreams();
 Assert.assertEquals(1, inputKeys.size());
 Assert.assertTrue(inputKeys.contains("testavro.SIMPLE1"));
 Assert.assertEquals(2, outputKeys.size());
 Assert.assertTrue(outputKeys.contains("testavro.COMPLEX1"));
 Assert.assertTrue(outputKeys.contains("testavro.Profile"));
 Assert.assertEquals(2, outputStreamList.size());
 Assert.assertEquals("testavro.COMPLEX1", outputStreamList.get(0));
 Assert.assertEquals("testavro.Profile", outputStreamList.get(1));
}
origin: apache/samza

@Test
public void testGetInputAndOutputStreamConfigsFanIn() {
 List<String> sqlStmts = Arrays.asList("Insert into testavro.COMPLEX1 select * from testavro.SIMPLE1",
   "insert into testavro.COMPLEX1 select * from testavro.SIMPLE2");
 Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10);
 config.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
 List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
 SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream)
     .collect(Collectors.toList()),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
 Set<String> inputKeys = samzaSqlApplicationConfig.getInputSystemStreamConfigBySource().keySet();
 Set<String> outputKeys = samzaSqlApplicationConfig.getOutputSystemStreamConfigsBySource().keySet();
 List<String> outputStreamList = samzaSqlApplicationConfig.getOutputSystemStreams();
 Assert.assertEquals(2, inputKeys.size());
 Assert.assertTrue(inputKeys.contains("testavro.SIMPLE1"));
 Assert.assertTrue(inputKeys.contains("testavro.SIMPLE2"));
 Assert.assertEquals(1, outputKeys.size());
 Assert.assertTrue(outputKeys.contains("testavro.COMPLEX1"));
 Assert.assertEquals(2, outputStreamList.size());
 Assert.assertEquals("testavro.COMPLEX1", outputStreamList.get(0));
 Assert.assertEquals("testavro.COMPLEX1", outputStreamList.get(1));
}
origin: apache/samza

 private void sendToOutputStream(String queryLogicalId, String logicalOpId, String sinkStream, StreamApplicationDescriptor appDesc, TranslatorContext translatorContext, RelNode node, int queryId) {
  SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(sinkStream);
  MessageStream<SamzaSqlRelMessage> stream = translatorContext.getMessageStream(node.getId());
  MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(queryLogicalId, logicalOpId, sinkStream, queryId));
  Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor();
  if (!tableDescriptor.isPresent()) {
   KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>());
   String systemName = sinkConfig.getSystemName();
   DelegatingSystemDescriptor
     sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new);
   GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamId(), noOpKVSerde);
   OutputStream stm = outputMsgStreams.computeIfAbsent(sinkConfig.getSource(), v -> appDesc.getOutputStream(osd));
   outputStream.sendTo(stm);
  } else {
   Table outputTable = appDesc.getTable(tableDescriptor.get());
   if (outputTable == null) {
    String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource();
    throw new SamzaException(msg);
   }
   outputStream.sendTo(outputTable);
  }
 }
}
origin: org.apache.samza/samza-sql

 private void sendToOutputStream(StreamApplicationDescriptor appDesc, TranslatorContext context, RelNode node, int queryId) {
  SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG);
  MessageStream<SamzaSqlRelMessage> stream = context.getMessageStream(node.getId());
  MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG, queryId));
  Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor();
  if (!tableDescriptor.isPresent()) {
   KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>());
   String systemName = sinkConfig.getSystemName();
   DelegatingSystemDescriptor
     sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new);
   GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamName(), noOpKVSerde);
   if (OutputMapFunction.logOutputStream == null) {
    OutputMapFunction.logOutputStream = appDesc.getOutputStream(osd);
   }
   outputStream.sendTo(OutputMapFunction.logOutputStream);
  } else {
   Table outputTable = appDesc.getTable(tableDescriptor.get());
   if (outputTable == null) {
    String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource();
    throw new SamzaException(msg);
   }
   outputStream.sendTo(outputTable);
  }
 }
}
org.apache.samza.sql.runnerSamzaSqlApplicationConfiggetOutputSystemStreamConfigsBySource

Popular methods of SamzaSqlApplicationConfig

  • <init>
  • createIOResolver
  • getInputSystemStreamConfigBySource
  • getUdfMetadata
  • initializePlugin
  • createUdfResolver
  • deserializeSqlStmts
  • getDomainProperties
  • getOutputSystemStreams
  • getRelSchemaProviders
  • getSamzaRelConverters
  • getSystemStreamName
  • getSamzaRelConverters,
  • getSystemStreamName,
  • getWindowDurationMs,
  • populateSystemStreams,
  • populateSystemStreamsAndGetRelRoots,
  • serializeSqlStmts,
  • getIoResolver,
  • getMetadataTopicPrefix,
  • getSamzaRelTableKeyConverters

Popular in Java

  • Updating database using SQL prepared statement
  • getSystemService (Context)
  • getOriginalFilename (MultipartFile)
    Return the original filename in the client's filesystem.This may contain path information depending
  • orElseThrow (Optional)
    Return the contained value, if present, otherwise throw an exception to be created by the provided s
  • SQLException (java.sql)
    An exception that indicates a failed JDBC operation. It provides the following information about pro
  • Calendar (java.util)
    Calendar is an abstract base class for converting between a Date object and a set of integer fields
  • Map (java.util)
    A Map is a data structure consisting of a set of keys and values in which each key is mapped to a si
  • TreeSet (java.util)
    TreeSet is an implementation of SortedSet. All optional operations (adding and removing) are support
  • Semaphore (java.util.concurrent)
    A counting semaphore. Conceptually, a semaphore maintains a set of permits. Each #acquire blocks if
  • ReentrantLock (java.util.concurrent.locks)
    A reentrant mutual exclusion Lock with the same basic behavior and semantics as the implicit monitor
  • Top 12 Jupyter Notebook extensions
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now