public static <K, V> KVSerde<K, V> of(Serde<K> keySerde, Serde<V> valueSerde) { return new KVSerde<>(keySerde, valueSerde); }
public static <K, V> KVSerde<K, V> of(Serde<K> keySerde, Serde<V> valueSerde) { return new KVSerde<>(keySerde, valueSerde); }
private RocksDbTableDescriptor createTableDescriptor() { return new RocksDbTableDescriptor(TABLE_ID, new KVSerde(new NoOpSerde(), new NoOpSerde())); } }
private InMemoryTableDescriptor createTableDescriptor() { return new InMemoryTableDescriptor(TABLE_ID, new KVSerde(new NoOpSerde(), new NoOpSerde())); } }
private LocalTableDescriptor createTableDescriptor() { return new MockLocalTableDescriptor(TABLE_ID, new KVSerde(new StringSerde(), new IntegerSerde())); }
intermediateStream1, new KVSerde<>(new NoOpSerde(), new NoOpSerde())); GenericOutputDescriptor<KV<Object, Object>> intermediateOutput1 = system1Descriptor.getOutputDescriptor( intermediateStream1, new KVSerde<>(new NoOpSerde(), new NoOpSerde())); intermediateBroadcast, new KVSerde<>(new NoOpSerde<>(), new NoOpSerde<>())); GenericOutputDescriptor<KV<Object, Object>> intermediateBroacastOutput1 = system1Descriptor.getOutputDescriptor( intermediateBroadcast, new KVSerde<>(new NoOpSerde<>(), new NoOpSerde<>())); inputDescriptors.put(intermediateStream1, intermediateInput1); outputDescriptors.put(intermediateStream1, intermediateOutput1);
private StreamOperatorTask createStreamOperatorTask(Clock clock, StreamApplicationDescriptorImpl graphSpec) throws Exception { Map<String, String> mapConfig = new HashMap<>(); mapConfig.put("job.name", "jobName"); mapConfig.put("job.id", "jobId"); StreamTestUtils.addStreamConfigs(mapConfig, "inStream", "insystem", "instream"); StreamTestUtils.addStreamConfigs(mapConfig, "inStream2", "insystem", "instream2"); Context context = new MockContext(new MapConfig(mapConfig)); TaskModel taskModel = mock(TaskModel.class); when(taskModel.getSystemStreamPartitions()).thenReturn(ImmutableSet .of(new SystemStreamPartition("insystem", "instream", new Partition(0)), new SystemStreamPartition("insystem", "instream2", new Partition(0)))); when(context.getTaskContext().getTaskModel()).thenReturn(taskModel); when(context.getTaskContext().getTaskMetricsRegistry()).thenReturn(new MetricsRegistryMap()); when(context.getContainerContext().getContainerMetricsRegistry()).thenReturn(new MetricsRegistryMap()); // need to return different stores for left and right side IntegerSerde integerSerde = new IntegerSerde(); TimestampedValueSerde timestampedValueSerde = new TimestampedValueSerde(new KVSerde(integerSerde, integerSerde)); when(context.getTaskContext().getStore(eq("jobName-jobId-join-j1-L"))) .thenReturn(new TestInMemoryStore(integerSerde, timestampedValueSerde)); when(context.getTaskContext().getStore(eq("jobName-jobId-join-j1-R"))) .thenReturn(new TestInMemoryStore(integerSerde, timestampedValueSerde)); StreamOperatorTask sot = new StreamOperatorTask(graphSpec.getOperatorSpecGraph(), clock); sot.init(context); return sot; }
private StreamApplicationDescriptorImpl createStreamGraphWithStreamTableJoinAndSendToSameTable() { /** * A special example of stream-table join where a stream is joined with a table, and the result is * sent to the same table. This example is necessary to ensure {@link ExecutionPlanner} does not * get stuck traversing the virtual cycle between stream-table-join and send-to-table operator specs * indefinitely. * * The reason such virtual cycle is present is to support computing partitions of intermediate * streams participating in stream-table joins. Please, refer to SAMZA SEP-16 for more details. */ return new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor); TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor( "table-id", new KVSerde(new StringSerde(), new StringSerde())); Table table = appDesc.getTable(tableDescriptor); messageStream1 .join(table, mock(StreamTableJoinFunction.class)) .sendTo(table); }, config); }
private StreamApplicationDescriptorImpl createStreamGraphWithStreamTableJoinWithSideInputs() { /** * Example stream-table join where table t is configured with input1 (64) as a side-input stream. * * join-table t -> output1 (8) * | * input2 (16) -> partitionBy ("64") __| * */ return new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor); OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor); TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor( "table-id", new KVSerde(new StringSerde(), new StringSerde())) .withSideInputs(Arrays.asList("input1")) .withSideInputsProcessor(mock(SideInputsProcessor.class)); Table table = appDesc.getTable(tableDescriptor); messageStream2 .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1") .join(table, mock(StreamTableJoinFunction.class)) .sendTo(output1); }, config); }
private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamTableJoin() { /** * Example stream-table join that is invalid due to disagreement in partition count * between the 2 input streams. * * input1 (64) -> send-to-table t * * join-table t -> output1 (8) * | * input2 (16) ————————— * */ return new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor); MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor); OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor); TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor( "table-id", new KVSerde(new StringSerde(), new StringSerde())); Table table = appDesc.getTable(tableDescriptor); messageStream1.sendTo(table); messageStream1 .join(table, mock(StreamTableJoinFunction.class)) .join(messageStream2, mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(1), "j2") .sendTo(output1); }, config); }
private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamTableJoinWithSideInputs() { /** * Example stream-table join that is invalid due to disagreement in partition count between the * stream behind table t and another joined stream. Table t is configured with input2 (16) as * side-input stream. * * join-table t -> output1 (8) * | * input1 (64) ————————— * */ return new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor); OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor); TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor( "table-id", new KVSerde(new StringSerde(), new StringSerde())) .withSideInputs(Arrays.asList("input2")) .withSideInputsProcessor(mock(SideInputsProcessor.class)); Table table = appDesc.getTable(tableDescriptor); messageStream1 .join(table, mock(StreamTableJoinFunction.class)) .sendTo(output1); }, config); }
"table-id", new KVSerde(new StringSerde(), new StringSerde())); Table table = appDesc.getTable(tableDescriptor);
output2Spec = new StreamSpec("output2", "output2", "system2"); KVSerde<Object, Object> kvSerde = new KVSerde<>(new NoOpSerde(), new NoOpSerde()); String mockSystemFactoryClass = "factory.class.name"; system1Descriptor = new GenericSystemDescriptor("system1", mockSystemFactoryClass);
KVSerde<Object, Object> kvSerde = new KVSerde<>(new NoOpSerde(), new NoOpSerde()); String mockSystemFactoryClass = "factory.class.name"; GenericSystemDescriptor system1 = new GenericSystemDescriptor("system1", mockSystemFactoryClass);