@Override public <StreamMessageType> KafkaInputDescriptor<StreamMessageType> getInputDescriptor(String streamId, Serde<StreamMessageType> serde) { return new KafkaInputDescriptor<>(streamId, this, serde, null); }
@Override public Map<String, String> toConfig() { HashMap<String, String> configs = new HashMap<>(super.toConfig()); // Note: Kafka configuration needs the topic's physical name, not the stream-id. // We won't have that here if user only specified it in configs, or if it got rewritten // by the planner to something different than what's in this descriptor. String streamName = getPhysicalName().orElse(getStreamId()); String systemName = getSystemName(); consumerAutoOffsetResetOptional.ifPresent(autoOffsetReset -> configs.put(String.format(CONSUMER_AUTO_OFFSET_RESET_CONFIG_KEY, systemName, streamName), autoOffsetReset)); consumerFetchMessageMaxBytesOptional.ifPresent(fetchMessageMaxBytes -> configs.put(String.format(CONSUMER_FETCH_MESSAGE_MAX_BYTES_CONFIG_KEY, systemName, streamName), Long.toString(fetchMessageMaxBytes))); return configs; } }
@Test public void testISDConfigsWithOverrides() { KafkaSystemDescriptor sd = new KafkaSystemDescriptor("kafka"); KafkaInputDescriptor<KV<String, Integer>> isd = sd.getInputDescriptor("input-stream", KVSerde.of(new StringSerde(), new IntegerSerde())) .withConsumerAutoOffsetReset("largest") .withConsumerFetchMessageMaxBytes(1024 * 1024); Map<String, String> generatedConfigs = isd.toConfig();; assertEquals("kafka", generatedConfigs.get("streams.input-stream.samza.system")); assertEquals("largest", generatedConfigs.get("systems.kafka.streams.input-stream.consumer.auto.offset.reset")); assertEquals("1048576", generatedConfigs.get("systems.kafka.streams.input-stream.consumer.fetch.message.max.bytes")); }
@Test public void testISDConfigsWithDefaults() { KafkaSystemDescriptor sd = new KafkaSystemDescriptor("kafka") .withConsumerZkConnect(ImmutableList.of("localhost:123")) .withProducerBootstrapServers(ImmutableList.of("localhost:567", "localhost:890")); KafkaInputDescriptor<KV<String, Integer>> isd = sd.getInputDescriptor("input-stream", KVSerde.of(new StringSerde(), new IntegerSerde())); Map<String, String> generatedConfigs = isd.toConfig(); assertEquals("kafka", generatedConfigs.get("streams.input-stream.samza.system")); assertEquals(1, generatedConfigs.size()); // verify that there are no other configs } }
@Override public Map<String, String> toConfig() { HashMap<String, String> configs = new HashMap<>(super.toConfig()); // Note: Kafka configuration needs the topic's physical name, not the stream-id. // We won't have that here if user only specified it in configs, or if it got rewritten // by the planner to something different than what's in this descriptor. String streamName = getPhysicalName().orElse(getStreamId()); String systemName = getSystemName(); consumerAutoOffsetResetOptional.ifPresent(autoOffsetReset -> configs.put(String.format(CONSUMER_AUTO_OFFSET_RESET_CONFIG_KEY, systemName, streamName), autoOffsetReset)); consumerFetchMessageMaxBytesOptional.ifPresent(fetchMessageMaxBytes -> configs.put(String.format(CONSUMER_FETCH_MESSAGE_MAX_BYTES_CONFIG_KEY, systemName, streamName), Long.toString(fetchMessageMaxBytes))); return configs; } }
@Override public Map<String, String> toConfig() { HashMap<String, String> configs = new HashMap<>(super.toConfig()); // Note: Kafka configuration needs the topic's physical name, not the stream-id. // We won't have that here if user only specified it in configs, or if it got rewritten // by the planner to something different than what's in this descriptor. String streamName = getPhysicalName().orElse(getStreamId()); String systemName = getSystemName(); consumerAutoOffsetResetOptional.ifPresent(autoOffsetReset -> configs.put(String.format(CONSUMER_AUTO_OFFSET_RESET_CONFIG_KEY, systemName, streamName), autoOffsetReset)); consumerFetchMessageMaxBytesOptional.ifPresent(fetchMessageMaxBytes -> configs.put(String.format(CONSUMER_FETCH_MESSAGE_MAX_BYTES_CONFIG_KEY, systemName, streamName), Long.toString(fetchMessageMaxBytes))); return configs; } }
@Override public <StreamMessageType> KafkaInputDescriptor<StreamMessageType> getInputDescriptor(String streamId, Serde<StreamMessageType> serde) { return new KafkaInputDescriptor<>(streamId, this, serde, null); }
@Override public <StreamMessageType> KafkaInputDescriptor<StreamMessageType> getInputDescriptor(String streamId, Serde<StreamMessageType> serde) { return new KafkaInputDescriptor<>(streamId, this, serde, null); }