congrats Icon
New! Announcing Tabnine Chat Beta
Learn More
Tabnine Logo
StreamApplicationDescriptor
Code IndexAdd Tabnine to your IDE (free)

How to use
StreamApplicationDescriptor
in
org.apache.samza.application.descriptors

Best Java code snippets using org.apache.samza.application.descriptors.StreamApplicationDescriptor (Showing top 20 results out of 315)

origin: apache/samza

private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamStreamJoin() {
 /**
  * Creates the following stream-stream join which is invalid due to partition count disagreement
  * between the 2 input streams.
  *
  *   input1 (64) --
  *                 |
  *                join -> output1 (8)
  *                 |
  *   input3 (32) --
  */
 return new StreamApplicationDescriptorImpl(appDesc -> {
   MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
   MessageStream<KV<Object, Object>> messageStream3 = appDesc.getInputStream(input3Descriptor);
   OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
   messageStream1
     .join(messageStream3,
       mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(2), "j1")
     .sendTo(output1);
  }, config);
}
origin: apache/samza

 private void sendToOutputStream(String queryLogicalId, String logicalOpId, String sinkStream, StreamApplicationDescriptor appDesc, TranslatorContext translatorContext, RelNode node, int queryId) {
  SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(sinkStream);
  MessageStream<SamzaSqlRelMessage> stream = translatorContext.getMessageStream(node.getId());
  MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(queryLogicalId, logicalOpId, sinkStream, queryId));
  Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor();
  if (!tableDescriptor.isPresent()) {
   KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>());
   String systemName = sinkConfig.getSystemName();
   DelegatingSystemDescriptor
     sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new);
   GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamId(), noOpKVSerde);
   OutputStream stm = outputMsgStreams.computeIfAbsent(sinkConfig.getSource(), v -> appDesc.getOutputStream(osd));
   outputStream.sendTo(stm);
  } else {
   Table outputTable = appDesc.getTable(tableDescriptor.get());
   if (outputTable == null) {
    String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource();
    throw new SamzaException(msg);
   }
   outputStream.sendTo(outputTable);
  }
 }
}
origin: apache/samza

try {
 List<String> dslStmts = SamzaSqlDslConverter.fetchSqlFromConfig(appDescriptor.getConfig());
   SamzaSqlApplicationConfig.populateSystemStreamsAndGetRelRoots(dslStmts, appDescriptor.getConfig(),
     inputSystemStreams, outputSystemStreams);
   new SamzaSqlApplicationConfig(appDescriptor.getConfig(), inputSystemStreams, outputSystemStreams);
 appDescriptor.withApplicationTaskContextFactory(new ApplicationTaskContextFactory<SamzaSqlApplicationContext>() {
  @Override
  public SamzaSqlApplicationContext create(ExternalContext externalContext, JobContext jobContext,
origin: apache/samza

@Test(expected = IllegalStateException.class)
public void testSetDefaultSystemDescriptorAfterGettingOutputStream() {
 String streamId = "test-stream-1";
 GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, mock(Serde.class));
 new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.getOutputStream(osd);
   appDesc.withDefaultSystem(sd); // should throw exception
  }, getConfig());
}
origin: apache/samza

public ExampleExpandingSystemDescriptor(String systemName) {
 super(systemName, FACTORY_CLASS_NAME,
   (InputTransformer<String>) IncomingMessageEnvelope::toString,
   (streamGraph, inputDescriptor) -> (MessageStream<Long>) streamGraph.getInputStream(inputDescriptor)
 );
}
origin: apache/samza

MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
MessageStream<KV<Object, Object>> messageStream3 = appDesc.getInputStream(input3Descriptor);
OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
Table table = appDesc.getTable(tableDescriptor);
origin: apache/samza

@Test
public void testMultipleSystemDescriptorForSameSystemName() {
 GenericSystemDescriptor sd1 = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericSystemDescriptor sd2 = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericInputDescriptor isd1 = sd1.getInputDescriptor("test-stream-1", mock(Serde.class));
 GenericInputDescriptor isd2 = sd2.getInputDescriptor("test-stream-2", mock(Serde.class));
 GenericOutputDescriptor osd1 = sd2.getOutputDescriptor("test-stream-3", mock(Serde.class));
 new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.getInputStream(isd1);
   try {
    appDesc.getInputStream(isd2);
    fail("Adding input stream with the same system name but different SystemDescriptor should have failed");
   } catch (IllegalStateException e) { }
   try {
    appDesc.getOutputStream(osd1);
    fail("adding output stream with the same system name but different SystemDescriptor should have failed");
   } catch (IllegalStateException e) { }
  }, getConfig());
 new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.withDefaultSystem(sd2);
   try {
    appDesc.getInputStream(isd1);
    fail("Adding input stream with the same system name as the default system but different SystemDescriptor should have failed");
   } catch (IllegalStateException e) { }
  }, getConfig());
}
origin: apache/samza

private StreamApplicationDescriptorImpl createStreamGraphWithStreamTableJoinAndSendToSameTable() {
 /**
  * A special example of stream-table join where a stream is joined with a table, and the result is
  * sent to the same table. This example is necessary to ensure {@link ExecutionPlanner} does not
  * get stuck traversing the virtual cycle between stream-table-join and send-to-table operator specs
  * indefinitely.
  *
  * The reason such virtual cycle is present is to support computing partitions of intermediate
  * streams participating in stream-table joins. Please, refer to SAMZA SEP-16 for more details.
  */
 return new StreamApplicationDescriptorImpl(appDesc -> {
   MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
   TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor(
    "table-id", new KVSerde(new StringSerde(), new StringSerde()));
   Table table = appDesc.getTable(tableDescriptor);
   messageStream1
    .join(table, mock(StreamTableJoinFunction.class))
    .sendTo(table);
  }, config);
}
origin: apache/samza

@Test(expected = IllegalStateException.class)
public void testSetDefaultSystemDescriptorAfterGettingInputStream() {
 String streamId = "test-stream-1";
 GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mock(Serde.class));
 new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.getInputStream(isd);
   appDesc.withDefaultSystem(sd); // should throw exception
  }, getConfig());
}
origin: apache/samza

@Test
public void testGetTable() throws Exception {
 Config mockConfig = getConfig();
 String tableId = "t1";
 BaseTableDescriptor mockTableDescriptor = mock(BaseTableDescriptor.class);
 when(mockTableDescriptor.getTableId()).thenReturn(tableId);
 AtomicReference<TableImpl> table = new AtomicReference<>();
 StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
   table.set((TableImpl) appDesc.getTable(mockTableDescriptor));
  }, mockConfig);
 assertEquals(tableId, table.get().getTableId());
}
origin: apache/samza

@Test(expected = IllegalStateException.class)
public void testGetSameOutputStreamTwice() {
 String streamId = "test-stream-1";
 GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericOutputDescriptor osd1 = sd.getOutputDescriptor(streamId, mock(Serde.class));
 GenericOutputDescriptor osd2 = sd.getOutputDescriptor(streamId, mock(Serde.class));
 new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.getOutputStream(osd1);
   appDesc.getOutputStream(osd2); // should throw exception
  }, getConfig());
}
origin: apache/samza

/**
 * For unit testing only
 */
@VisibleForTesting
void translate(SamzaSqlQueryParser.QueryInfo queryInfo, StreamApplicationDescriptor appDesc, int queryId) {
 QueryPlanner planner =
   new QueryPlanner(sqlConfig.getRelSchemaProviders(), sqlConfig.getInputSystemStreamConfigBySource(),
     sqlConfig.getUdfMetadata());
 final RelRoot relRoot = planner.plan(queryInfo.getSelectQuery());
 SamzaSqlExecutionContext executionContext = new SamzaSqlExecutionContext(sqlConfig);
 TranslatorContext translatorContext = new TranslatorContext(appDesc, relRoot, executionContext);
 translate(relRoot, sqlConfig.getOutputSystemStreams().get(queryId), translatorContext, queryId);
 Map<Integer, TranslatorContext> translatorContexts = new HashMap<>();
 translatorContexts.put(queryId, translatorContext.clone());
 appDesc.withApplicationTaskContextFactory(new ApplicationTaskContextFactory<SamzaSqlApplicationContext>() {
  @Override
  public SamzaSqlApplicationContext create(ExternalContext externalContext, JobContext jobContext,
    ContainerContext containerContext, TaskContext taskContext,
    ApplicationContainerContext applicationContainerContext) {
   return new SamzaSqlApplicationContext(translatorContexts);
  }
 });
}
origin: apache/samza

@Test
public void testApplicationContainerContextFactory() {
 ApplicationContainerContextFactory factory = mock(ApplicationContainerContextFactory.class);
 StreamApplication testApp = appDesc -> appDesc.withApplicationContainerContextFactory(factory);
 StreamApplicationDescriptorImpl appSpec = new StreamApplicationDescriptorImpl(testApp, getConfig());
 assertEquals(appSpec.getApplicationContainerContextFactory(), Optional.of(factory));
}
origin: apache/samza

StreamApplication getRepartitionOnlyStreamApplication() {
 return appDesc -> {
  MessageStream<KV<String, Object>> input1 = appDesc.getInputStream(input1Descriptor);
  input1.partitionBy(KV::getKey, KV::getValue, mock(KVSerde.class), "p1");
 };
}
origin: apache/samza

private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamTableJoin() {
 /**
  * Example stream-table join that is invalid due to disagreement in partition count
  * between the 2 input streams.
  *
  *    input1 (64) -> send-to-table t
  *
  *                   join-table t -> output1 (8)
  *                         |
  *    input2 (16) —————————
  *
  */
 return new StreamApplicationDescriptorImpl(appDesc -> {
   MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
   MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
   OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
   TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor(
    "table-id", new KVSerde(new StringSerde(), new StringSerde()));
   Table table = appDesc.getTable(tableDescriptor);
   messageStream1.sendTo(table);
   messageStream1
     .join(table, mock(StreamTableJoinFunction.class))
     .join(messageStream2,
       mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(1), "j2")
     .sendTo(output1);
  }, config);
}
origin: apache/samza

@Test(expected = IllegalStateException.class)
public void testGetTableWithBadId() {
 Config mockConfig = getConfig();
 new StreamApplicationDescriptorImpl(appDesc -> {
   BaseTableDescriptor mockTableDescriptor = mock(BaseTableDescriptor.class);
   when(mockTableDescriptor.getTableId()).thenReturn("my.table");
   appDesc.getTable(mockTableDescriptor);
  }, mockConfig);
}
origin: apache/samza

@Test
public void testGetOutputStreamWithKeyValueSerde() {
 String streamId = "test-stream-1";
 KVSerde mockKVSerde = mock(KVSerde.class);
 Serde mockKeySerde = mock(Serde.class);
 Serde mockValueSerde = mock(Serde.class);
 doReturn(mockKeySerde).when(mockKVSerde).getKeySerde();
 doReturn(mockValueSerde).when(mockKVSerde).getValueSerde();
 GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, mockKVSerde);
 StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.getOutputStream(osd);
  }, getConfig());
 OutputStreamImpl<TestMessageEnvelope> outputStreamImpl = streamAppDesc.getOutputStreams().get(streamId);
 assertEquals(streamId, outputStreamImpl.getStreamId());
 assertEquals(osd, streamAppDesc.getOutputDescriptors().get(streamId));
 assertEquals(mockKeySerde, outputStreamImpl.getKeySerde());
 assertEquals(mockValueSerde, outputStreamImpl.getValueSerde());
}
origin: apache/samza

@Test
public void testApplicationTaskContextFactory() {
 ApplicationTaskContextFactory factory = mock(ApplicationTaskContextFactory.class);
 StreamApplication testApp = appDesc -> appDesc.withApplicationTaskContextFactory(factory);
 StreamApplicationDescriptorImpl appSpec = new StreamApplicationDescriptorImpl(testApp, getConfig());
 assertEquals(appSpec.getApplicationTaskContextFactory(), Optional.of(factory));
}
origin: apache/samza

private StreamApplicationDescriptorImpl createSimpleGraph() {
 /**
  * a simple graph of partitionBy and map
  *
  * input1 -> partitionBy -> map -> output1
  *
  */
 return new StreamApplicationDescriptorImpl(appDesc-> {
   MessageStream<KV<Object, Object>> input1 = appDesc.getInputStream(input1Descriptor);
   OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
   input1
     .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1")
     .map(kv -> kv)
     .sendTo(output1);
  }, config);
}
origin: apache/samza

 StreamApplication getBroadcastOnlyStreamApplication(Serde serde) {
  return appDesc -> {
   MessageStream<KV<String, Object>> input = appDesc.getInputStream(input1Descriptor);
   input.broadcast(serde, "b1");
  };
 }
}
org.apache.samza.application.descriptorsStreamApplicationDescriptor

Javadoc

A StreamApplicationDescriptor contains the description of inputs, outputs, state, configuration and the processing logic for a Samza High Level API StreamApplication.

Use the StreamApplicationDescriptor obtained from StreamApplication#describe to get the MessageStreams, OutputStreams and Tables corresponding to their respective InputDescriptors, OutputDescriptors and TableDescriptors.

Use the MessageStream API operators to describe the processing logic for the StreamApplication.

Most used methods

  • getInputStream
    Gets the input MessageStream corresponding to the inputDescriptor. A MessageStream, obtained by call
  • getOutputStream
    Gets the OutputStream corresponding to the outputDescriptor. An OutputStream>, obtained by calling t
  • getTable
    Gets the Table corresponding to the TableDescriptor. Multiple invocations of this method with the sa
  • withApplicationTaskContextFactory
  • getConfig
  • withApplicationContainerContextFactory
  • withDefaultSystem
  • withProcessorLifecycleListenerFactory

Popular in Java

  • Making http requests using okhttp
  • setScale (BigDecimal)
  • requestLocationUpdates (LocationManager)
  • getSupportFragmentManager (FragmentActivity)
  • BigInteger (java.math)
    An immutable arbitrary-precision signed integer.FAST CRYPTOGRAPHY This implementation is efficient f
  • Socket (java.net)
    Provides a client-side TCP socket.
  • Scanner (java.util)
    A parser that parses a text string of primitive types and strings with the help of regular expressio
  • StringTokenizer (java.util)
    Breaks a string into tokens; new code should probably use String#split.> // Legacy code: StringTo
  • JList (javax.swing)
  • SAXParseException (org.xml.sax)
    Encapsulate an XML parse error or warning.> This module, both source code and documentation, is in t
  • Top Sublime Text plugins
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now