public Serde<String> getSerde(String name, Config config) { return new StringSerde(config.get("encoding", "UTF-8")); } }
public Serde<String> getSerde(String name, Config config) { return new StringSerde(config.get("encoding", "UTF-8")); } }
private void addValueSerde(Map<String, String> map) { String serdeId = "value-serde"; map.put(String.format(SerializerConfig.SERDE_SERIALIZED_INSTANCE(), serdeId), serializeSerde(new StringSerde("UTF-8"))); map.put(String.format(JavaTableConfig.STORE_MSG_SERDE, TABLE_ID), serdeId); }
private StreamApplicationDescriptorImpl createStreamGraphWithStreamTableJoinAndSendToSameTable() { /** * A special example of stream-table join where a stream is joined with a table, and the result is * sent to the same table. This example is necessary to ensure {@link ExecutionPlanner} does not * get stuck traversing the virtual cycle between stream-table-join and send-to-table operator specs * indefinitely. * * The reason such virtual cycle is present is to support computing partitions of intermediate * streams participating in stream-table joins. Please, refer to SAMZA SEP-16 for more details. */ return new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor); TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor( "table-id", new KVSerde(new StringSerde(), new StringSerde())); Table table = appDesc.getTable(tableDescriptor); messageStream1 .join(table, mock(StreamTableJoinFunction.class)) .sendTo(table); }, config); }
private LocalTableDescriptor createTableDescriptor() { return new MockLocalTableDescriptor(TABLE_ID, new KVSerde(new StringSerde(), new IntegerSerde())); }
private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamTableJoinWithSideInputs() { /** * Example stream-table join that is invalid due to disagreement in partition count between the * stream behind table t and another joined stream. Table t is configured with input2 (16) as * side-input stream. * * join-table t -> output1 (8) * | * input1 (64) ————————— * */ return new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor); OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor); TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor( "table-id", new KVSerde(new StringSerde(), new StringSerde())) .withSideInputs(Arrays.asList("input2")) .withSideInputsProcessor(mock(SideInputsProcessor.class)); Table table = appDesc.getTable(tableDescriptor); messageStream1 .join(table, mock(StreamTableJoinFunction.class)) .sendTo(output1); }, config); }
private StreamApplicationDescriptorImpl createStreamGraphWithStreamTableJoinWithSideInputs() { /** * Example stream-table join where table t is configured with input1 (64) as a side-input stream. * * join-table t -> output1 (8) * | * input2 (16) -> partitionBy ("64") __| * */ return new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor); OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor); TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor( "table-id", new KVSerde(new StringSerde(), new StringSerde())) .withSideInputs(Arrays.asList("input1")) .withSideInputsProcessor(mock(SideInputsProcessor.class)); Table table = appDesc.getTable(tableDescriptor); messageStream2 .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1") .join(table, mock(StreamTableJoinFunction.class)) .sendTo(output1); }, config); }
@Test public void testWithInputOutputStreams() { String systemName = "system-name"; String streamId1 = "input-stream1"; String streamId2 = "input-stream2"; String streamId3 = "output-stream1"; String streamId4 = "output-stream2"; EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName); systemDescriptor.getInputDescriptor(streamId1, "entity-namespace1", "entity1", new StringSerde()); systemDescriptor.getInputDescriptor(streamId2, "entity-namespace2", "entity2", new StringSerde()); systemDescriptor.getOutputDescriptor(streamId3, "entity-namespace3", "entity3", new StringSerde()); systemDescriptor.getOutputDescriptor(streamId4, "entity-namespace4", "entity4", new StringSerde()); Map<String, String> generatedConfigs = systemDescriptor.toConfig(); assertEquals("org.apache.samza.system.eventhub.EventHubSystemFactory", generatedConfigs.get(String.format("systems.%s.samza.factory", systemName))); assertEquals(streamId1 + "," + streamId2 + "," + streamId3 + "," + streamId4, generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName))); assertEquals(2, generatedConfigs.size()); } }
private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamTableJoin() { /** * Example stream-table join that is invalid due to disagreement in partition count * between the 2 input streams. * * input1 (64) -> send-to-table t * * join-table t -> output1 (8) * | * input2 (16) ————————— * */ return new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor); MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor); OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor); TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor( "table-id", new KVSerde(new StringSerde(), new StringSerde())); Table table = appDesc.getTable(tableDescriptor); messageStream1.sendTo(table); messageStream1 .join(table, mock(StreamTableJoinFunction.class)) .join(messageStream2, mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(1), "j2") .sendTo(output1); }, config); }
@Test public void testWithoutEntityConnectionConfigs() { String systemName = "eventHub"; String streamId = "input-stream"; EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName); EventHubsInputDescriptor<KV<String, String>> inputDescriptor = systemDescriptor .getInputDescriptor(streamId, "entity-namespace", "entity3", new StringSerde()); Map<String, String> generatedConfigs = inputDescriptor.toConfig(); assertEquals("eventHub", generatedConfigs.get("streams.input-stream.samza.system")); assertEquals("entity-namespace", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamId))); assertEquals("entity3", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamId))); assertNull(generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamId))); assertNull(generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamId))); assertNull(generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_CONSUMER_GROUP, streamId))); assertEquals(3, generatedConfigs.size()); // verify that there are no other configs }
@Test public void testWithoutEntityConnectionConfigs() { String systemName = "eventHub"; String streamId = "output-stream"; EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName); EventHubsOutputDescriptor<KV<String, String>> outputDescriptor = systemDescriptor .getOutputDescriptor(streamId, "entity-namespace", "entity3", new StringSerde()); Map<String, String> generatedConfigs = outputDescriptor.toConfig(); assertEquals("eventHub", generatedConfigs.get("streams.output-stream.samza.system")); assertEquals("entity-namespace", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamId))); assertEquals("entity3", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamId))); assertNull(generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamId))); assertNull(generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamId))); assertNull(generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_CONSUMER_GROUP, streamId))); assertEquals(3, generatedConfigs.size()); // verify that there are no other configs }
@Test public void testStringTimeSeriesKey() { TimeSeriesKey<String> storeKey = new TimeSeriesKey<>("test", 1, 23); TimeSeriesKeySerde<String> serde = new TimeSeriesKeySerde<>(new StringSerde("UTF-8")); byte[] serializedBytes = serde.toBytes(storeKey); TimeSeriesKey<String> deserializedTimeSeriesKey = serde.fromBytes(serializedBytes); assertEquals(storeKey.getKey(), deserializedTimeSeriesKey.getKey()); assertEquals(storeKey.getSeqNum(), deserializedTimeSeriesKey.getSeqNum()); assertEquals(storeKey.getTimestamp(), deserializedTimeSeriesKey.getTimestamp()); assertEquals(storeKey, deserializedTimeSeriesKey); }
"table-id", new KVSerde(new StringSerde(), new StringSerde())); Table table = appDesc.getTable(tableDescriptor);
@Test public void testNullTimeSeriesKey() { TimeSeriesKey<String> storeKey = new TimeSeriesKey<>(null, 1, 23); TimeSeriesKeySerde<String> serde = new TimeSeriesKeySerde<>(new StringSerde("UTF-8")); byte[] serializedBytes = serde.toBytes(storeKey); TimeSeriesKey<String> deserializedTimeSeriesKey = serde.fromBytes(serializedBytes); assertEquals(storeKey.getKey(), deserializedTimeSeriesKey.getKey()); assertEquals(storeKey.getSeqNum(), deserializedTimeSeriesKey.getSeqNum()); assertEquals(storeKey.getTimestamp(), deserializedTimeSeriesKey.getTimestamp()); assertEquals(storeKey, deserializedTimeSeriesKey); }
@Test public void testEntityConnectionConfigs() { String systemName = "eventHub"; String streamId = "output-stream"; EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName); EventHubsOutputDescriptor<KV<String, String>> outputDescriptor = systemDescriptor .getOutputDescriptor(streamId, "entity-namespace", "entity3", new StringSerde()) .withSasKeyName("secretkey") .withSasKey("sasToken-123"); Map<String, String> generatedConfigs = outputDescriptor.toConfig(); assertEquals("eventHub", generatedConfigs.get("streams.output-stream.samza.system")); assertEquals("entity-namespace", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamId))); assertEquals("entity3", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamId))); assertEquals("secretkey", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamId))); assertEquals("sasToken-123", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamId))); }
@Test public void testGetWithNonExistentKeys() { TimeSeriesStore<String, byte[]> timeSeriesStore = newTimeSeriesStore(new StringSerde("UTF-8"), true); timeSeriesStore.put("hello", "world-1".getBytes(), 1L); // read from a non-existent key List<TimestampedValue<byte[]>> values = readStore(timeSeriesStore, "non-existent-key", 0, Integer.MAX_VALUE); Assert.assertEquals(0, values.size()); // read from an existing key but out of range timestamp values = readStore(timeSeriesStore, "hello", 2, Integer.MAX_VALUE); Assert.assertEquals(0, values.size()); }
@Test public void testEntityConnectionConfigs() { String systemName = "eventHub"; String streamId = "input-stream"; EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName); EventHubsInputDescriptor<KV<String, String>> inputDescriptor = systemDescriptor .getInputDescriptor(streamId, "entity-namespace", "entity3", new StringSerde()) .withSasKeyName("secretkey") .withSasKey("sasToken-123") .withConsumerGroup("$notdefault"); Map<String, String> generatedConfigs = inputDescriptor.toConfig(); assertEquals("eventHub", generatedConfigs.get("streams.input-stream.samza.system")); assertEquals("entity-namespace", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamId))); assertEquals("entity3", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamId))); assertEquals("secretkey", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamId))); assertEquals("sasToken-123", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamId))); assertEquals("$notdefault", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_CONSUMER_GROUP, streamId))); }
@Test public void testInputOperatorSpec() { Serde<Object> objSerde = new Serde<Object>() { @Override public Object fromBytes(byte[] bytes) { return null; } @Override public byte[] toBytes(Object object) { return new byte[0]; } }; InputOperatorSpec inputOperatorSpec = new InputOperatorSpec( "mockStreamId", new StringSerde("UTF-8"), objSerde, null, true, "op0"); InputOperatorSpec inputOpCopy = (InputOperatorSpec) OperatorSpecTestUtils.copyOpSpec(inputOperatorSpec); assertNotEquals("Expected deserialized copy of operator spec should not be the same as the original operator spec", inputOperatorSpec, inputOpCopy); assertTrue(inputOperatorSpec.isClone(inputOpCopy)); }
@Test public void testDeletesInOverwriteMode() { // instantiate a store in overwrite mode TimeSeriesStore<String, byte[]> timeSeriesStore = newTimeSeriesStore(new StringSerde("UTF-8"), false); // insert an entry with key "hello" at timestamps "1" and "2" timeSeriesStore.put("hello", "world-1".getBytes(), 1L); timeSeriesStore.put("hello", "world-1".getBytes(), 2L); timeSeriesStore.put("hello", "world-2".getBytes(), 2L); List<TimestampedValue<byte[]>> values = readStore(timeSeriesStore, "hello", 1L, 3L); Assert.assertEquals(2, values.size()); timeSeriesStore.remove("hello", 0L, 3L); values = readStore(timeSeriesStore, "hello", 1L, 3L); Assert.assertEquals(0, values.size()); }
@Test public void testWithDescriptorOverrides() { String systemName = "system-name"; String streamId1 = "input-stream1"; String streamId2 = "input-stream2"; String streamId3 = "output-stream1"; String streamId4 = "output-stream2"; EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName) .withMaxEventCountPerPoll(1000) .withNumClientThreads(5) .withPartitioningMethod(PartitioningMethod.PARTITION_KEY_AS_PARTITION) .withPrefetchCount(100) .withReceiveQueueSize(500) .withRuntimeInfoTimeout(60000) .withSendKeys(false); systemDescriptor.getInputDescriptor(streamId1, "entity-namespace1", "entity1", new StringSerde()); systemDescriptor.getInputDescriptor(streamId2, "entity-namespace2", "entity2", new StringSerde()); systemDescriptor.getOutputDescriptor(streamId3, "entity-namespace3", "entity3", new StringSerde()); systemDescriptor.getOutputDescriptor(streamId4, "entity-namespace4", "entity4", new StringSerde()); Map<String, String> generatedConfigs = systemDescriptor.toConfig(); assertEquals("org.apache.samza.system.eventhub.EventHubSystemFactory", generatedConfigs.get(String.format("systems.%s.samza.factory", systemName))); assertEquals("1000", generatedConfigs.get(String.format(EventHubConfig.CONFIG_MAX_EVENT_COUNT_PER_POLL, systemName))); assertEquals("5", generatedConfigs.get(String.format(EventHubConfig.CONFIG_SYSTEM_NUM_CLIENT_THREADS, systemName))); assertEquals("PARTITION_KEY_AS_PARTITION", generatedConfigs.get(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName))); assertEquals("100", generatedConfigs.get(String.format(EventHubConfig.CONFIG_PREFETCH_COUNT, systemName))); assertEquals("500", generatedConfigs.get(String.format(EventHubConfig.CONFIG_CONSUMER_BUFFER_CAPACITY, systemName))); assertEquals("60000", generatedConfigs.get(String.format(EventHubConfig.CONFIG_FETCH_RUNTIME_INFO_TIMEOUT_MILLIS, systemName))); assertEquals("false", generatedConfigs.get(String.format(EventHubConfig.CONFIG_SEND_KEY_IN_EVENT_PROPERTIES, systemName))); assertEquals(streamId1 + "," + streamId2 + "," + streamId3 + "," + streamId4, generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName))); }