How to use

getOutputStream

method

in

org.apache.samza.application.descriptors.StreamApplicationDescriptor

@Test(expected = IllegalStateException.class)
public void testGetSameOutputStreamTwice() {
 String streamId = "test-stream-1";
 GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericOutputDescriptor osd1 = sd.getOutputDescriptor(streamId, mock(Serde.class));
 GenericOutputDescriptor osd2 = sd.getOutputDescriptor(streamId, mock(Serde.class));
 new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.getOutputStream(osd1);
   appDesc.getOutputStream(osd2); // should throw exception
  }, getConfig());
}

 private void sendToOutputStream(String queryLogicalId, String logicalOpId, String sinkStream, StreamApplicationDescriptor appDesc, TranslatorContext translatorContext, RelNode node, int queryId) {
  SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(sinkStream);
  MessageStream<SamzaSqlRelMessage> stream = translatorContext.getMessageStream(node.getId());
  MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(queryLogicalId, logicalOpId, sinkStream, queryId));
  Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor();
  if (!tableDescriptor.isPresent()) {
   KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>());
   String systemName = sinkConfig.getSystemName();
   DelegatingSystemDescriptor
     sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new);
   GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamId(), noOpKVSerde);
   OutputStream stm = outputMsgStreams.computeIfAbsent(sinkConfig.getSource(), v -> appDesc.getOutputStream(osd));
   outputStream.sendTo(stm);
  } else {
   Table outputTable = appDesc.getTable(tableDescriptor.get());
   if (outputTable == null) {
    String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource();
    throw new SamzaException(msg);
   }
   outputStream.sendTo(outputTable);
  }
 }
}

private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamStreamJoin() {
 /**
  * Creates the following stream-stream join which is invalid due to partition count disagreement
  * between the 2 input streams.
  *
  *   input1 (64) --
  *                 |
  *                join -> output1 (8)
  *                 |
  *   input3 (32) --
  */
 return new StreamApplicationDescriptorImpl(appDesc -> {
   MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
   MessageStream<KV<Object, Object>> messageStream3 = appDesc.getInputStream(input3Descriptor);
   OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
   messageStream1
     .join(messageStream3,
       mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(2), "j1")
     .sendTo(output1);
  }, config);
}

private StreamApplicationDescriptorImpl createStreamGraphWithJoinAndWindow() {
 return new StreamApplicationDescriptorImpl(appDesc -> {
   MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor).map(m -> m);
   MessageStream<KV<Object, Object>> messageStream2 =
    appDesc.getInputStream(input2Descriptor)
      .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1")
      .filter(m -> true);
   MessageStream<KV<Object, Object>> messageStream3 =
    appDesc.getInputStream(input3Descriptor)
      .filter(m -> true)
      .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p2")
      .map(m -> m);
   OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
   OutputStream<KV<Object, Object>> output2 = appDesc.getOutputStream(output2Descriptor);
   messageStream1.map(m -> m)
     .filter(m -> true)
     .window(Windows.keyedTumblingWindow(m -> m, Duration.ofMillis(8), (Serde<KV<Object, Object>>) mock(Serde.class), (Serde<KV<Object, Object>>) mock(Serde.class)), "w1");
   messageStream2.map(m -> m)
     .filter(m -> true)
     .window(Windows.keyedTumblingWindow(m -> m, Duration.ofMillis(16), (Serde<KV<Object, Object>>) mock(Serde.class), (Serde<KV<Object, Object>>) mock(Serde.class)), "w2");
   messageStream1.join(messageStream2, mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofMillis(1600), "j1").sendTo(output1);
   messageStream3.join(messageStream2, mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofMillis(100), "j2").sendTo(output2);
   messageStream3.join(messageStream2, mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofMillis(252), "j3").sendTo(output2);
  }, config);
}

 private void sendToOutputStream(StreamApplicationDescriptor appDesc, TranslatorContext context, RelNode node, int queryId) {
  SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG);
  MessageStream<SamzaSqlRelMessage> stream = context.getMessageStream(node.getId());
  MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG, queryId));
  Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor();
  if (!tableDescriptor.isPresent()) {
   KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>());
   String systemName = sinkConfig.getSystemName();
   DelegatingSystemDescriptor
     sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new);
   GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamName(), noOpKVSerde);
   if (OutputMapFunction.logOutputStream == null) {
    OutputMapFunction.logOutputStream = appDesc.getOutputStream(osd);
   }
   outputStream.sendTo(OutputMapFunction.logOutputStream);
  } else {
   Table outputTable = appDesc.getTable(tableDescriptor.get());
   if (outputTable == null) {
    String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource();
    throw new SamzaException(msg);
   }
   outputStream.sendTo(outputTable);
  }
 }
}

@Test(expected = IllegalStateException.class)
public void testSetDefaultSystemDescriptorAfterGettingOutputStream() {
 String streamId = "test-stream-1";
 GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, mock(Serde.class));
 new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.getOutputStream(osd);
   appDesc.withDefaultSystem(sd); // should throw exception
  }, getConfig());
}

 OutputStream stm = outputMsgStreams.computeIfAbsent(source, v -> streamAppDesc.getOutputStream(osd));
 outputStream.sendTo(stm);
} else {

private StreamApplicationDescriptorImpl createSimpleGraph() {
 /**
  * a simple graph of partitionBy and map
  *
  * input1 -> partitionBy -> map -> output1
  *
  */
 return new StreamApplicationDescriptorImpl(appDesc-> {
   MessageStream<KV<Object, Object>> input1 = appDesc.getInputStream(input1Descriptor);
   OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
   input1
     .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1")
     .map(kv -> kv)
     .sendTo(output1);
  }, config);
}

@Test
public void testGetOutputStreamWithKeyValueSerde() {
 String streamId = "test-stream-1";
 KVSerde mockKVSerde = mock(KVSerde.class);
 Serde mockKeySerde = mock(Serde.class);
 Serde mockValueSerde = mock(Serde.class);
 doReturn(mockKeySerde).when(mockKVSerde).getKeySerde();
 doReturn(mockValueSerde).when(mockKVSerde).getValueSerde();
 GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, mockKVSerde);
 StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.getOutputStream(osd);
  }, getConfig());
 OutputStreamImpl<TestMessageEnvelope> outputStreamImpl = streamAppDesc.getOutputStreams().get(streamId);
 assertEquals(streamId, outputStreamImpl.getStreamId());
 assertEquals(osd, streamAppDesc.getOutputDescriptors().get(streamId));
 assertEquals(mockKeySerde, outputStreamImpl.getKeySerde());
 assertEquals(mockValueSerde, outputStreamImpl.getValueSerde());
}

@Test(expected = IllegalArgumentException.class)
public void testGetOutputStreamWithNullSerde() {
 String streamId = "test-stream-1";
 GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, null);
 new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.getOutputStream(osd);
  }, getConfig());
}

    .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p2")
    .map(m -> m);
OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
OutputStream<KV<Object, Object>> output2 = appDesc.getOutputStream(output2Descriptor);

@Test
public void testMaxPartitionLimit() {
 int partitionLimit = IntermediateStreamManager.MAX_INFERRED_PARTITIONS;
 ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
 StreamApplicationDescriptorImpl graphSpec = new StreamApplicationDescriptorImpl(appDesc -> {
   MessageStream<KV<Object, Object>> input1 = appDesc.getInputStream(input4Descriptor);
   OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
   input1.partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1").map(kv -> kv).sendTo(output1);
  }, config);
 JobGraph jobGraph = (JobGraph) planner.plan(graphSpec);
 // Partitions should be the same as input1
 jobGraph.getIntermediateStreams().forEach(edge -> {
   assertEquals(partitionLimit, edge.getPartitionCount()); // max of input1 and output1
  });
}

GenericOutputDescriptor outputDescriptor = sd.getOutputDescriptor(outputStreamId, mock(Serde.class));
MessageStream<Object> inputStream = appDesc.getInputStream(inputDescriptor);
OutputStream<Object> outputStream = appDesc.getOutputStream(outputDescriptor);

@Test
public void testGetOutputStreamWithValueSerde() {
 String streamId = "test-stream-1";
 Serde mockValueSerde = mock(Serde.class);
 GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, mockValueSerde);
 StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.getOutputStream(osd);
  }, getConfig());
 OutputStreamImpl<TestMessageEnvelope> outputStreamImpl = streamAppDesc.getOutputStreams().get(streamId);
 assertEquals(streamId, outputStreamImpl.getStreamId());
 assertEquals(osd, streamAppDesc.getOutputDescriptors().get(streamId));
 assertTrue(outputStreamImpl.getKeySerde() instanceof NoOpSerde);
 assertEquals(mockValueSerde, outputStreamImpl.getValueSerde());
}

StreamApplication getRepartitionJoinStreamApplication() {
 return appDesc -> {
  MessageStream<KV<String, Object>> input1 = appDesc.getInputStream(input1Descriptor);
  MessageStream<KV<String, Object>> input2 = appDesc.getInputStream(input2Descriptor);
  OutputStream<KV<String, Object>> output = appDesc.getOutputStream(outputDescriptor);
  JoinFunction<String, Object, Object, KV<String, Object>> mockJoinFn = mock(JoinFunction.class);
  input1
    .partitionBy(KV::getKey, KV::getValue, defaultSerde, "p1")
    .map(kv -> kv.value)
    .join(input2.map(kv -> kv.value), mockJoinFn,
      new StringSerde(), new JsonSerdeV2<>(Object.class), new JsonSerdeV2<>(Object.class),
      Duration.ofHours(1), "j1")
    .sendTo(output);
 };
}

@Test
public void testMultipleSystemDescriptorForSameSystemName() {
 GenericSystemDescriptor sd1 = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericSystemDescriptor sd2 = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericInputDescriptor isd1 = sd1.getInputDescriptor("test-stream-1", mock(Serde.class));
 GenericInputDescriptor isd2 = sd2.getInputDescriptor("test-stream-2", mock(Serde.class));
 GenericOutputDescriptor osd1 = sd2.getOutputDescriptor("test-stream-3", mock(Serde.class));
 new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.getInputStream(isd1);
   try {
    appDesc.getInputStream(isd2);
    fail("Adding input stream with the same system name but different SystemDescriptor should have failed");
   } catch (IllegalStateException e) { }
   try {
    appDesc.getOutputStream(osd1);
    fail("adding output stream with the same system name but different SystemDescriptor should have failed");
   } catch (IllegalStateException e) { }
  }, getConfig());
 new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.withDefaultSystem(sd2);
   try {
    appDesc.getInputStream(isd1);
    fail("Adding input stream with the same system name as the default system but different SystemDescriptor should have failed");
   } catch (IllegalStateException e) { }
  }, getConfig());
}

MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
MessageStream<KV<Object, Object>> messageStream3 = appDesc.getInputStream(input3Descriptor);
OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);

private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamTableJoin() {
 /**
  * Example stream-table join that is invalid due to disagreement in partition count
  * between the 2 input streams.
  *
  *    input1 (64) -> send-to-table t
  *
  *                   join-table t -> output1 (8)
  *                         |
  *    input2 (16) —————————
  *
  */
 return new StreamApplicationDescriptorImpl(appDesc -> {
   MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
   MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
   OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
   TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor(
    "table-id", new KVSerde(new StringSerde(), new StringSerde()));
   Table table = appDesc.getTable(tableDescriptor);
   messageStream1.sendTo(table);
   messageStream1
     .join(table, mock(StreamTableJoinFunction.class))
     .join(messageStream2,
       mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(1), "j2")
     .sendTo(output1);
  }, config);
}

private StreamApplicationDescriptorImpl createStreamGraphWithStreamTableJoinWithSideInputs() {
 /**
  * Example stream-table join where table t is configured with input1 (64) as a side-input stream.
  *
  *                                   join-table t -> output1 (8)
  *                                        |
  *    input2 (16) -> partitionBy ("64") __|
  *
  */
 return new StreamApplicationDescriptorImpl(appDesc -> {
   MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
   OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
   TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor(
    "table-id", new KVSerde(new StringSerde(), new StringSerde()))
     .withSideInputs(Arrays.asList("input1"))
     .withSideInputsProcessor(mock(SideInputsProcessor.class));
   Table table = appDesc.getTable(tableDescriptor);
   messageStream2
     .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1")
     .join(table, mock(StreamTableJoinFunction.class))
     .sendTo(output1);
  }, config);
}

private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamTableJoinWithSideInputs() {
 /**
  * Example stream-table join that is invalid due to disagreement in partition count between the
  * stream behind table t and another joined stream. Table t is configured with input2 (16) as
  * side-input stream.
  *
  *                   join-table t -> output1 (8)
  *                         |
  *    input1 (64) —————————
  *
  */
 return new StreamApplicationDescriptorImpl(appDesc -> {
   MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
   OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
   TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor(
    "table-id", new KVSerde(new StringSerde(), new StringSerde()))
     .withSideInputs(Arrays.asList("input2"))
     .withSideInputsProcessor(mock(SideInputsProcessor.class));
   Table table = appDesc.getTable(tableDescriptor);
   messageStream1
     .join(table, mock(StreamTableJoinFunction.class))
     .sendTo(output1);
  }, config);
}