public ExampleExpandingSystemDescriptor(String systemName) { super(systemName, FACTORY_CLASS_NAME, (InputTransformer<String>) IncomingMessageEnvelope::toString, (streamGraph, inputDescriptor) -> (MessageStream<Long>) streamGraph.getInputStream(inputDescriptor) ); }
StreamApplication getRepartitionOnlyStreamApplication() { return appDesc -> { MessageStream<KV<String, Object>> input1 = appDesc.getInputStream(input1Descriptor); input1.partitionBy(KV::getKey, KV::getValue, mock(KVSerde.class), "p1"); }; }
StreamApplication getBroadcastOnlyStreamApplication(Serde serde) { return appDesc -> { MessageStream<KV<String, Object>> input = appDesc.getInputStream(input1Descriptor); input.broadcast(serde, "b1"); }; } }
@Test(expected = IllegalStateException.class) public void testGetSameInputStreamTwice() { String streamId = "test-stream-1"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd1 = sd.getInputDescriptor(streamId, mock(Serde.class)); GenericInputDescriptor isd2 = sd.getInputDescriptor(streamId, mock(Serde.class)); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd1); // should throw exception appDesc.getInputStream(isd2); }, getConfig()); }
@Test(expected = IllegalArgumentException.class) public void testScheduledFunctionAsValueFn() { ScheduledMapFn valueFn = new ScheduledMapFn(); new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<Object> inputStream = appDesc.getInputStream(testInputDescriptor); inputStream.partitionBy(m -> m.toString(), valueFn, mock(KVSerde.class), "parByKey"); }, getConfig()); }
@Test(expected = IllegalArgumentException.class) public void testWatermarkFunctionAsValueFn() { WatermarkMapFn valueFn = new WatermarkMapFn(); new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<Object> inputStream = appDesc.getInputStream(testInputDescriptor); inputStream.partitionBy(m -> m.toString(), valueFn, mock(KVSerde.class), "parByKey"); }, getConfig()); }
@Test(expected = IllegalArgumentException.class) public void testWatermarkFunctionAsKeyFn() { WatermarkMapFn keyFn = new WatermarkMapFn(); new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<Object> inputStream = appDesc.getInputStream(testInputDescriptor); inputStream.partitionBy(keyFn, m -> m, mock(KVSerde.class), "parByKey"); }, getConfig()); }
@Test(expected = IllegalArgumentException.class) public void testScheduledFunctionAsKeyFn() { ScheduledMapFn keyFn = new ScheduledMapFn(); new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<Object> inputStream = appDesc.getInputStream(testInputDescriptor); inputStream.partitionBy(keyFn, m -> m, mock(KVSerde.class), "parByKey"); }, getConfig()); }
@Test public void testCopy() { StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { MessageStream inputStream = appDesc.getInputStream(testInputDescriptor); inputStream.partitionBy(m -> m.toString(), m -> m, mock(KVSerde.class), testRepartitionedStreamName); }, getConfig()); OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); OperatorSpecGraph clonedGraph = specGraph.clone(); OperatorSpecTestUtils.assertClonedGraph(specGraph, clonedGraph); }
@Test(expected = IllegalStateException.class) public void testSetDefaultSystemDescriptorAfterGettingInputStream() { String streamId = "test-stream-1"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mock(Serde.class)); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); appDesc.withDefaultSystem(sd); // should throw exception }, getConfig()); }
private StreamApplicationDescriptorImpl createSimpleGraph() { /** * a simple graph of partitionBy and map * * input1 -> partitionBy -> map -> output1 * */ return new StreamApplicationDescriptorImpl(appDesc-> { MessageStream<KV<Object, Object>> input1 = appDesc.getInputStream(input1Descriptor); OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor); input1 .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1") .map(kv -> kv) .sendTo(output1); }, config); }
StreamApplication getRepartitionJoinStreamApplication() { return appDesc -> { MessageStream<KV<String, Object>> input1 = appDesc.getInputStream(input1Descriptor); MessageStream<KV<String, Object>> input2 = appDesc.getInputStream(input2Descriptor); OutputStream<KV<String, Object>> output = appDesc.getOutputStream(outputDescriptor); JoinFunction<String, Object, Object, KV<String, Object>> mockJoinFn = mock(JoinFunction.class); input1 .partitionBy(KV::getKey, KV::getValue, defaultSerde, "p1") .map(kv -> kv.value) .join(input2.map(kv -> kv.value), mockJoinFn, new StringSerde(), new JsonSerdeV2<>(Object.class), new JsonSerdeV2<>(Object.class), Duration.ofHours(1), "j1") .sendTo(output); }; }
@Test(expected = IllegalArgumentException.class) public void testGetInputStreamWithNullSerde() { GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd = sd.getInputDescriptor("mockStreamId", null); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); }, getConfig()); }
@Test public void testGetInputStreamWithRelaxedTypes() { String streamId = "test-stream-1"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mock(Serde.class)); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); }, getConfig()); InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(streamId); assertEquals(OpCode.INPUT, inputOpSpec.getOpCode()); assertEquals(streamId, inputOpSpec.getStreamId()); assertEquals(isd, streamAppDesc.getInputDescriptors().get(streamId)); }
@Test public void testMaxPartitionLimit() { int partitionLimit = IntermediateStreamManager.MAX_INFERRED_PARTITIONS; ExecutionPlanner planner = new ExecutionPlanner(config, streamManager); StreamApplicationDescriptorImpl graphSpec = new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<KV<Object, Object>> input1 = appDesc.getInputStream(input4Descriptor); OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor); input1.partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1").map(kv -> kv).sendTo(output1); }, config); JobGraph jobGraph = (JobGraph) planner.plan(graphSpec); // Partitions should be the same as input1 jobGraph.getIntermediateStreams().forEach(edge -> { assertEquals(partitionLimit, edge.getPartitionCount()); // max of input1 and output1 }); }
@Test public void testGetInputStreamWithTransformFunction() { String streamId = "test-stream-1"; Serde mockValueSerde = mock(Serde.class); InputTransformer transformer = ime -> ime; MockTransformingSystemDescriptor sd = new MockTransformingSystemDescriptor("mockSystem", transformer); MockInputDescriptor isd = sd.getInputDescriptor(streamId, mockValueSerde); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); }, getConfig()); InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(streamId); assertEquals(OpCode.INPUT, inputOpSpec.getOpCode()); assertEquals(streamId, inputOpSpec.getStreamId()); assertEquals(isd, streamAppDesc.getInputDescriptors().get(streamId)); assertEquals(transformer, inputOpSpec.getTransformer()); }
@Test public void testGetInputStreamWithValueSerde() { String streamId = "test-stream-1"; Serde mockValueSerde = mock(Serde.class); GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mockValueSerde); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); }, getConfig()); InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(streamId); assertEquals(OpCode.INPUT, inputOpSpec.getOpCode()); assertEquals(streamId, inputOpSpec.getStreamId()); assertEquals(isd, streamAppDesc.getInputDescriptors().get(streamId)); assertTrue(inputOpSpec.getKeySerde() instanceof NoOpSerde); assertEquals(mockValueSerde, inputOpSpec.getValueSerde()); }
private StreamApplicationDescriptorImpl getKeyedSessionWindowStreamGraph(AccumulationMode mode, Duration duration) throws IOException { StreamApplication userApp = appDesc -> { KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde()); GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde); appDesc.getInputStream(inputDescriptor) .window(Windows.keyedSessionWindow(KV::getKey, duration, new IntegerSerde(), kvSerde) .setAccumulationMode(mode), "w1") .sink((message, messageCollector, taskCoordinator) -> { SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream"); messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message)); }); }; return new StreamApplicationDescriptorImpl(userApp, config); }
private StreamApplicationDescriptorImpl getTumblingWindowStreamGraph(AccumulationMode mode, Duration duration, Trigger<KV<Integer, Integer>> earlyTrigger) throws IOException { StreamApplication userApp = appDesc -> { KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde()); GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde); appDesc.getInputStream(inputDescriptor) .window(Windows.tumblingWindow(duration, kvSerde).setEarlyTrigger(earlyTrigger) .setAccumulationMode(mode), "w1") .sink((message, messageCollector, taskCoordinator) -> { SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream"); messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message)); }); }; return new StreamApplicationDescriptorImpl(userApp, config); }
private StreamApplicationDescriptorImpl getKeyedTumblingWindowStreamGraph(AccumulationMode mode, Duration duration, Trigger<KV<Integer, Integer>> earlyTrigger) throws IOException { StreamApplication userApp = appDesc -> { KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde()); GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde); appDesc.getInputStream(inputDescriptor) .window(Windows.keyedTumblingWindow(KV::getKey, duration, new IntegerSerde(), kvSerde) .setEarlyTrigger(earlyTrigger).setAccumulationMode(mode), "w1") .sink((message, messageCollector, taskCoordinator) -> { SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream"); messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message)); }); }; return new StreamApplicationDescriptorImpl(userApp, config); }