Tabnine Logo
SqlIOConfig
Code IndexAdd Tabnine to your IDE (free)

How to use
SqlIOConfig
in
org.apache.samza.sql.interfaces

Best Java code snippets using org.apache.samza.sql.interfaces.SqlIOConfig (Showing top 20 results out of 315)

origin: org.apache.samza/samza-sql

public SamzaSqlApplicationConfig(Config staticConfig, Set<String> inputSystemStreams,
  Set<String> outputSystemStreams) {
 ioResolver = createIOResolver(staticConfig);
 inputSystemStreamConfigBySource = inputSystemStreams.stream()
    .collect(Collectors.toMap(Function.identity(), src -> ioResolver.fetchSourceInfo(src)));
 outputSystemStreamConfigsBySource = outputSystemStreams.stream()
    .collect(Collectors.toMap(Function.identity(), x -> ioResolver.fetchSinkInfo(x)));
 systemStreamConfigsBySource = new HashMap<>(inputSystemStreamConfigBySource);
 systemStreamConfigsBySource.putAll(outputSystemStreamConfigsBySource);
 Set<SqlIOConfig> systemStreamConfigs = new HashSet<>(systemStreamConfigsBySource.values());
 relSchemaProvidersBySource = systemStreamConfigs.stream()
   .collect(Collectors.toMap(SqlIOConfig::getSource,
     x -> initializePlugin("RelSchemaProvider", x.getRelSchemaProviderName(), staticConfig,
       CFG_FMT_REL_SCHEMA_PROVIDER_DOMAIN,
       (o, c) -> ((RelSchemaProviderFactory) o).create(x.getSystemStream(), c))));
 samzaRelConvertersBySource = systemStreamConfigs.stream()
   .collect(Collectors.toMap(SqlIOConfig::getSource,
     x -> initializePlugin("SamzaRelConverter", x.getSamzaRelConverterName(), staticConfig,
       CFG_FMT_SAMZA_REL_CONVERTER_DOMAIN, (o, c) -> ((SamzaRelConverterFactory) o).create(x.getSystemStream(),
         relSchemaProvidersBySource.get(x.getSource()), c))));
 udfResolver = createUdfResolver(staticConfig);
 udfMetadata = udfResolver.getUdfs();
 windowDurationMs = staticConfig.getLong(CFG_GROUPBY_WINDOW_DURATION_MS, DEFAULT_GROUPBY_WINDOW_DURATION_MS);
 // remove the SqlIOConfigs of outputs whose system is "log" out of systemStreamConfigsBySource
 outputSystemStreamConfigsBySource.forEach((k, v) -> {
   if (k.split("\\.")[0].equals(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG)) {
     systemStreamConfigsBySource.remove(k);
   }
 });
}
origin: apache/samza

 private void sendToOutputStream(String queryLogicalId, String logicalOpId, String sinkStream, StreamApplicationDescriptor appDesc, TranslatorContext translatorContext, RelNode node, int queryId) {
  SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(sinkStream);
  MessageStream<SamzaSqlRelMessage> stream = translatorContext.getMessageStream(node.getId());
  MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(queryLogicalId, logicalOpId, sinkStream, queryId));
  Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor();
  if (!tableDescriptor.isPresent()) {
   KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>());
   String systemName = sinkConfig.getSystemName();
   DelegatingSystemDescriptor
     sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new);
   GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamId(), noOpKVSerde);
   OutputStream stm = outputMsgStreams.computeIfAbsent(sinkConfig.getSource(), v -> appDesc.getOutputStream(osd));
   outputStream.sendTo(stm);
  } else {
   Table outputTable = appDesc.getTable(tableDescriptor.get());
   if (outputTable == null) {
    String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource();
    throw new SamzaException(msg);
   }
   outputStream.sendTo(outputTable);
  }
 }
}
origin: apache/samza

newConfig.put(String.format(CFG_FMT_SAMZA_STREAM_SYSTEM, inputSystemStreamConfig.getStreamId()),
  inputSystemStreamConfig.getSystemName());
newConfig.putAll(inputSystemStreamConfig.getConfig());
newConfig.put(String.format(CFG_FMT_SAMZA_STREAM_SYSTEM, outputSystemStreamConfig.getStreamId()),
  outputSystemStreamConfig.getSystemName());
newConfig.putAll(outputSystemStreamConfig.getConfig());
origin: org.apache.samza/samza-sql

newConfig.put(String.format(CFG_FMT_SAMZA_STREAM_SYSTEM, inputSystemStreamConfig.getStreamName()),
  inputSystemStreamConfig.getSystemName());
newConfig.putAll(inputSystemStreamConfig.getConfig());
newConfig.put(String.format(CFG_FMT_SAMZA_STREAM_SYSTEM, outputSystemStreamConfig.getStreamName()),
  outputSystemStreamConfig.getSystemName());
newConfig.putAll(outputSystemStreamConfig.getConfig());
origin: org.apache.samza/samza-sql

 void translate(final TableScan tableScan, final TranslatorContext context,
   Map<String, DelegatingSystemDescriptor> systemDescriptors, Map<String, MessageStream<KV<Object, Object>>> inputMsgStreams) {
  StreamApplicationDescriptor streamAppDesc = context.getStreamAppDescriptor();
  List<String> tableNameParts = tableScan.getTable().getQualifiedName();
  String sourceName = SqlIOConfig.getSourceFromSourceParts(tableNameParts);

  Validate.isTrue(relMsgConverters.containsKey(sourceName), String.format("Unknown source %s", sourceName));
  SqlIOConfig sqlIOConfig = systemStreamConfig.get(sourceName);
  final String systemName = sqlIOConfig.getSystemName();
  final String streamName = sqlIOConfig.getStreamName();
  final String source = sqlIOConfig.getSource();

  KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>());
  DelegatingSystemDescriptor
    sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new);
  GenericInputDescriptor<KV<Object, Object>> isd = sd.getInputDescriptor(streamName, noOpKVSerde);

  MessageStream<KV<Object, Object>> inputStream = inputMsgStreams.computeIfAbsent(source, v -> streamAppDesc.getInputStream(isd));
  MessageStream<SamzaSqlRelMessage> samzaSqlRelMessageStream = inputStream.map(new ScanMapFunction(sourceName, queryId));
  context.registerMessageStream(tableScan.getId(), samzaSqlRelMessageStream);
 }
}
origin: apache/samza

@Override
public SqlSchema getTableSchema(ExecutionContext context, String tableName) {
 /**
  *  currently Shell works only for systems that has Avro schemas
  */
 lastErrorMsg = "";
 int execId = execIdSeq.incrementAndGet();
 Map<String, String> staticConfigs = fetchSamzaSqlConfig(execId, context);
 Config samzaSqlConfig = new MapConfig(staticConfigs);
 SqlSchema sqlSchema = null;
 try {
  SqlIOResolver ioResolver = SamzaSqlApplicationConfig.createIOResolver(samzaSqlConfig);
  SqlIOConfig sourceInfo = ioResolver.fetchSourceInfo(tableName);
  RelSchemaProvider schemaProvider =
      SamzaSqlApplicationConfig.initializePlugin("RelSchemaProvider", sourceInfo.getRelSchemaProviderName(),
          samzaSqlConfig, SamzaSqlApplicationConfig.CFG_FMT_REL_SCHEMA_PROVIDER_DOMAIN,
          (o, c) -> ((RelSchemaProviderFactory) o).create(sourceInfo.getSystemStream(), c));
  sqlSchema =  schemaProvider.getSqlSchema();
 } catch (SamzaException ex) {
  String msg = "getTableSchema failed with exception ";
  lastErrorMsg = msg + ex.toString();
  LOG.error(msg, ex);
 }
 return sqlSchema;
}
origin: apache/samza

private JoinInputNode.InputType getInputType(RelNode relNode, TranslatorContext context) {
 // NOTE: Any intermediate form of a join is always a stream. Eg: For the second level join of
 // stream-table-table join, the left side of the join is join output, which we always
 // assume to be a stream. The intermediate stream won't be an instance of EnumerableTableScan.
 // The join key(s) for the table could be an udf in which case the relNode would be LogicalProject.
 if (relNode instanceof EnumerableTableScan || relNode instanceof LogicalProject) {
  SqlIOConfig sourceTableConfig = resolveSourceConfigForTable(relNode, context);
  if (sourceTableConfig == null || !sourceTableConfig.getTableDescriptor().isPresent()) {
   return JoinInputNode.InputType.STREAM;
  } else if (sourceTableConfig.getTableDescriptor().get() instanceof RemoteTableDescriptor ||
    sourceTableConfig.getTableDescriptor().get() instanceof CachingTableDescriptor) {
   return JoinInputNode.InputType.REMOTE_TABLE;
  } else {
   return JoinInputNode.InputType.LOCAL_TABLE;
  }
 } else {
  return JoinInputNode.InputType.STREAM;
 }
}
origin: apache/samza

List<String> sourceParts = ssc.getSourceParts();
RelSchemaProvider relSchemaProvider = relSchemaProviders.get(ssc.getSource());
origin: apache/samza

return new SqlIOConfig(systemName, streamName, fetchSystemConfigs(systemName), tableDescriptor);
origin: apache/samza

public SqlIOConfig(String systemName, String streamName, List<String> sourceParts,
  Config systemConfig, TableDescriptor tableDescriptor) {
 HashMap<String, String> streamConfigs = new HashMap<>(systemConfig);
 this.source = getSourceFromSourceParts(sourceParts);
 this.sourceParts = sourceParts;
 this.systemStream = new SystemStream(systemName, streamName);
   String.format("%s is not set or empty for system %s", CFG_SAMZA_REL_CONVERTER, systemName));
 if (isRemoteTable()) {
  samzaRelTableKeyConverterName = streamConfigs.get(CFG_SAMZA_REL_TABLE_KEY_CONVERTER);
  Validate.notEmpty(samzaRelTableKeyConverterName,
 streamConfigs.remove(CFG_REL_SCHEMA_PROVIDER);
 if (!isRemoteTable()) {
origin: apache/samza

String getSourceName() {
 return SqlIOConfig.getSourceFromSourceParts(relNode.getTable().getQualifiedName());
}
origin: org.apache.samza/samza-sql

String targetName = SqlIOConfig.getSourceFromSourceParts(tableNameParts);
final String systemName = sinkConfig.getSystemName();
final String streamName = sinkConfig.getStreamName();
final String source = sinkConfig.getSource();
MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(targetName, queryId));
Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor();
if (!tableDescriptor.isPresent()) {
 OutputStream stm = outputMsgStreams.computeIfAbsent(source, v -> streamAppDesc.getOutputStream(osd));
 Table outputTable = streamAppDesc.getTable(tableDescriptor.get());
 if (outputTable == null) {
  String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource();
  throw new SamzaException(msg);
origin: org.apache.samza/samza-sql

private boolean isTable(RelNode relNode) {
 // NOTE: Any intermediate form of a join is always a stream. Eg: For the second level join of
 // stream-table-table join, the left side of the join is join output, which we always
 // assume to be a stream. The intermediate stream won't be an instance of EnumerableTableScan.
 if (relNode instanceof EnumerableTableScan) {
  return resolveSourceConfig(relNode).getTableDescriptor().isPresent();
 } else {
  return false;
 }
}
origin: org.apache.samza/samza-sql

List<String> sourceParts = ssc.getSourceParts();
RelSchemaProvider relSchemaProvider = relSchemaProviders.get(ssc.getSource());
origin: org.apache.samza/samza-sql

return new SqlIOConfig(systemName, streamName, fetchSystemConfigs(systemName), tableDescriptor);
origin: org.apache.samza/samza-sql

public SqlIOConfig(String systemName, String streamName, List<String> sourceParts,
  Config systemConfig, TableDescriptor tableDescriptor) {
 HashMap<String, String> streamConfigs = new HashMap<>(systemConfig);
 this.systemName = systemName;
 this.streamName = streamName;
 this.source = getSourceFromSourceParts(sourceParts);
 this.sourceParts = sourceParts;
 this.systemStream = new SystemStream(systemName, streamName);
 this.tableDescriptor = Optional.ofNullable(tableDescriptor);
 samzaRelConverterName = streamConfigs.get(CFG_SAMZA_REL_CONVERTER);
 Validate.notEmpty(samzaRelConverterName,
   String.format("%s is not set or empty for system %s", CFG_SAMZA_REL_CONVERTER, systemName));
 relSchemaProviderName = streamConfigs.get(CFG_REL_SCHEMA_PROVIDER);
 // Removing the Samza SQL specific configs to get the remaining Samza configs.
 streamConfigs.remove(CFG_SAMZA_REL_CONVERTER);
 streamConfigs.remove(CFG_REL_SCHEMA_PROVIDER);
 // Currently, only local table is supported. And it is assumed that all tables are local tables.
 if (tableDescriptor != null) {
  streamConfigs.put(String.format(StreamConfig.BOOTSTRAP_FOR_STREAM_ID(), streamName), "true");
  streamConfigs.put(String.format(StreamConfig.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID(), streamName), "oldest");
 }
 config = new MapConfig(streamConfigs);
}
origin: org.apache.samza/samza-sql

 private void sendToOutputStream(StreamApplicationDescriptor appDesc, TranslatorContext context, RelNode node, int queryId) {
  SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG);
  MessageStream<SamzaSqlRelMessage> stream = context.getMessageStream(node.getId());
  MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG, queryId));
  Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor();
  if (!tableDescriptor.isPresent()) {
   KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>());
   String systemName = sinkConfig.getSystemName();
   DelegatingSystemDescriptor
     sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new);
   GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamName(), noOpKVSerde);
   if (OutputMapFunction.logOutputStream == null) {
    OutputMapFunction.logOutputStream = appDesc.getOutputStream(osd);
   }
   outputStream.sendTo(OutputMapFunction.logOutputStream);
  } else {
   Table outputTable = appDesc.getTable(tableDescriptor.get());
   if (outputTable == null) {
    String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource();
    throw new SamzaException(msg);
   }
   outputStream.sendTo(outputTable);
  }
 }
}
origin: apache/samza

x -> initializePlugin("RelSchemaProvider", x.getRelSchemaProviderName(), staticConfig,
  CFG_FMT_REL_SCHEMA_PROVIDER_DOMAIN,
  (o, c) -> ((RelSchemaProviderFactory) o).create(x.getSystemStream(), c))));
x -> initializePlugin("SamzaRelConverter", x.getSamzaRelConverterName(), staticConfig,
  CFG_FMT_SAMZA_REL_CONVERTER_DOMAIN, (o, c) -> ((SamzaRelConverterFactory) o).create(x.getSystemStream(),
    relSchemaProvidersBySource.get(x.getSource()), c))));
x -> initializePlugin("SamzaRelTableKeyConverter", x.getSamzaRelTableKeyConverterName(),
  staticConfig, CFG_FMT_SAMZA_REL_TABLE_KEY_CONVERTER_DOMAIN,
  (o, c) -> ((SamzaRelTableKeyConverterFactory) o).create(x.getSystemStream(), c))));
origin: apache/samza

StreamApplicationDescriptor streamAppDesc = context.getStreamAppDescriptor();
List<String> tableNameParts = tableScan.getTable().getQualifiedName();
String sourceName = SqlIOConfig.getSourceFromSourceParts(tableNameParts);
final String systemName = sqlIOConfig.getSystemName();
final String streamId = sqlIOConfig.getStreamId();
final String source = sqlIOConfig.getSource();
final boolean isRemoteTable = sqlIOConfig.getTableDescriptor().isPresent() &&
  (sqlIOConfig.getTableDescriptor().get() instanceof RemoteTableDescriptor ||
    sqlIOConfig.getTableDescriptor().get() instanceof CachingTableDescriptor);
origin: org.apache.samza/samza-sql

 private Table loadLocalTable(boolean isTablePosOnRight, List<Integer> tableKeyIds, LogicalJoin join, TranslatorContext context) {
  RelNode relNode = isTablePosOnRight ? join.getRight() : join.getLeft();

  MessageStream<SamzaSqlRelMessage> relOutputStream = context.getMessageStream(relNode.getId());

  SqlIOConfig sourceConfig = resolveSourceConfig(relNode);

  if (!sourceConfig.getTableDescriptor().isPresent()) {
   String errMsg = "Failed to resolve table source in join operation: node=" + relNode;
   log.error(errMsg);
   throw new SamzaException(errMsg);
  }

  // Create a table backed by RocksDb store with the fields in the join condition as composite key and relational
  // message as the value. Send the messages from the input stream denoted as 'table' to the created table store.
  Table<KV<SamzaSqlCompositeKey, SamzaSqlRelMessage>> table =
    context.getStreamAppDescriptor().getTable(sourceConfig.getTableDescriptor().get());

  relOutputStream
    .map(m -> new KV(createSamzaSqlCompositeKey(m, tableKeyIds), m))
    .sendTo(table);

  return table;
 }
}
org.apache.samza.sql.interfacesSqlIOConfig

Javadoc

Configs associated with an IO resource. Both stream and table resources are supported.

Most used methods

  • <init>
  • getRelSchemaProviderName
  • getSystemStream
  • getTableDescriptor
  • getConfig
  • getSamzaRelConverterName
  • getSource
  • getSourceFromSourceParts
  • getSourceParts
  • getSystemName
  • getSamzaRelTableKeyConverterName
  • getStreamId
  • getSamzaRelTableKeyConverterName,
  • getStreamId,
  • getStreamName,
  • isRemoteTable

Popular in Java

  • Reading from database using SQL prepared statement
  • notifyDataSetChanged (ArrayAdapter)
  • getSupportFragmentManager (FragmentActivity)
  • onRequestPermissionsResult (Fragment)
  • Font (java.awt)
    The Font class represents fonts, which are used to render text in a visible way. A font provides the
  • InputStream (java.io)
    A readable source of bytes.Most clients will use input streams that read data from the file system (
  • LinkedList (java.util)
    Doubly-linked list implementation of the List and Dequeinterfaces. Implements all optional list oper
  • Scanner (java.util)
    A parser that parses a text string of primitive types and strings with the help of regular expressio
  • AtomicInteger (java.util.concurrent.atomic)
    An int value that may be updated atomically. See the java.util.concurrent.atomic package specificati
  • Loader (org.hibernate.loader)
    Abstract superclass of object loading (and querying) strategies. This class implements useful common
  • Best plugins for Eclipse
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now