/** * Construct an instance with the supplied streams configuration and * clean up configuration. * @param streamsConfig the streams configuration. * @param cleanupConfig the cleanup configuration. * @since 2.1.2. * @deprecated in favor of {@link #StreamsBuilderFactoryBean(KafkaStreamsConfiguration, CleanupConfig)}. */ @Deprecated public StreamsBuilderFactoryBean(Map<String, Object> streamsConfig, CleanupConfig cleanupConfig) { Assert.notNull(streamsConfig, STREAMS_CONFIG_MUST_NOT_BE_NULL); Assert.notNull(cleanupConfig, CLEANUP_CONFIG_MUST_NOT_BE_NULL); this.streamsConfig = new StreamsConfig(streamsConfig); this.cleanupConfig = cleanupConfig; }
public static void maybeConfigureInterceptorsStreams(final Properties streamsConfig) { if (hasMonitoringConsumerInterceptor() && hasMonitoringProducerInterceptor()) { streamsConfig.put(StreamsConfig.producerPrefix(ProducerConfig.INTERCEPTOR_CLASSES_CONFIG), PRODUCER_INTERCEPTOR); streamsConfig.put(StreamsConfig.mainConsumerPrefix(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG), CONSUMER_INTERCEPTOR); } }
public Map<String, Object> getMainConsumerConfigs(final String groupId, final String clientId) { final Map<String, Object> consumerProps = getCommonConsumerConfigs(); final Map<String, Object> mainConsumerProps = originalsWithPrefix(MAIN_CONSUMER_PREFIX); for (final Map.Entry<String, Object> entry: mainConsumerProps.entrySet()) { consumerProps.put(entry.getKey(), entry.getValue()); consumerProps.put(UPGRADE_FROM_CONFIG, getString(UPGRADE_FROM_CONFIG)); consumerProps.put(REPLICATION_FACTOR_CONFIG, getInt(REPLICATION_FACTOR_CONFIG)); consumerProps.put(APPLICATION_SERVER_CONFIG, getString(APPLICATION_SERVER_CONFIG)); consumerProps.put(NUM_STANDBY_REPLICAS_CONFIG, getInt(NUM_STANDBY_REPLICAS_CONFIG)); consumerProps.put(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, StreamsPartitionAssignor.class.getName()); consumerProps.put(WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG, getLong(WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG)); final AdminClientConfig adminClientDefaultConfig = new AdminClientConfig(getClientPropsWithPrefix(ADMIN_CLIENT_PREFIX, AdminClientConfig.configNames())); consumerProps.put(adminClientPrefix(AdminClientConfig.RETRIES_CONFIG), adminClientDefaultConfig.getInt(AdminClientConfig.RETRIES_CONFIG)); final Map<String, Object> topicProps = originalsWithPrefix(TOPIC_PREFIX, false); if (topicProps.containsKey(topicPrefix(TopicConfig.SEGMENT_INDEX_BYTES_CONFIG))) { final int segmentSize = Integer.parseInt(topicProps.get(topicPrefix(TopicConfig.SEGMENT_INDEX_BYTES_CONFIG)).toString()); final Map<String, Object> producerProps = getClientPropsWithPrefix(PRODUCER_PREFIX, ProducerConfig.configNames()); final int batchSize; if (producerProps.containsKey(ProducerConfig.BATCH_SIZE_CONFIG)) {
private Map<String, Object> getCommonConsumerConfigs() { final Map<String, Object> clientProvidedProps = getClientPropsWithPrefix(CONSUMER_PREFIX, ConsumerConfig.configNames()); checkIfUnexpectedUserSpecifiedConsumerConfig(clientProvidedProps, NON_CONFIGURABLE_CONSUMER_DEFAULT_CONFIGS); checkIfUnexpectedUserSpecifiedConsumerConfig(clientProvidedProps, NON_CONFIGURABLE_CONSUMER_EOS_CONFIGS); final Map<String, Object> consumerProps = new HashMap<>(eosEnabled ? CONSUMER_EOS_OVERRIDES : CONSUMER_DEFAULT_OVERRIDES); consumerProps.putAll(getClientCustomProps()); consumerProps.putAll(clientProvidedProps); // bootstrap.servers should be from StreamsConfig consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, originals().get(BOOTSTRAP_SERVERS_CONFIG)); return consumerProps; }
public static Properties configStreams(String configFile, String stateDir, String serviceID) throws IOException { Properties props = LoadConfigs.loadConfig(configFile); props.put(StreamsConfig.APPLICATION_ID_CONFIG, serviceID); props.put(StreamsConfig.STATE_DIR_CONFIG, stateDir); props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); // Recommended cloud configuration for Streams (basically, wait for longer before exiting if brokers disconnect) props.put(StreamsConfig.REPLICATION_FACTOR_CONFIG, 3); props.put(StreamsConfig.producerPrefix(ProducerConfig.RETRIES_CONFIG), 2147483647); props.put("producer.confluent.batch.expiry.ms", 9223372036854775807L); props.put(StreamsConfig.producerPrefix(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG), 300000); props.put(StreamsConfig.producerPrefix(ProducerConfig.MAX_BLOCK_MS_CONFIG), 9223372036854775807L); return props; } }
/** * Get the configs for the {@link KafkaProducer producer}. * Properties using the prefix {@link #PRODUCER_PREFIX} will be used in favor over their non-prefixed versions * except in the case of {@link ProducerConfig#BOOTSTRAP_SERVERS_CONFIG} where we always use the non-prefixed * version as we only support reading/writing from/to the same Kafka Cluster. * * @param clientId clientId * @return Map of the producer configuration. */ @SuppressWarnings("WeakerAccess") public Map<String, Object> getProducerConfigs(final String clientId) { final Map<String, Object> clientProvidedProps = getClientPropsWithPrefix(PRODUCER_PREFIX, ProducerConfig.configNames()); checkIfUnexpectedUserSpecifiedConsumerConfig(clientProvidedProps, NON_CONFIGURABLE_PRODUCER_EOS_CONFIGS); // generate producer configs from original properties and overridden maps final Map<String, Object> props = new HashMap<>(eosEnabled ? PRODUCER_EOS_OVERRIDES : PRODUCER_DEFAULT_OVERRIDES); props.putAll(getClientCustomProps()); props.putAll(clientProvidedProps); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, originals().get(BOOTSTRAP_SERVERS_CONFIG)); // add client id with stream client id prefix props.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientId + "-producer"); return props; }
@Test public void testStreamsBuilderFactoryWithConfigProvidedLater() { Map<String, Object> props = new HashMap<>(); props.put(StreamsConfig.APPLICATION_ID_CONFIG, APPLICATION_ID); props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, this.brokerAddresses); StreamsConfig streamsConfig = new StreamsConfig(props); streamsBuilderFactoryBean.setStreamsConfig(streamsConfig); assertThat(streamsBuilderFactoryBean.isRunning()).isFalse(); streamsBuilderFactoryBean.start(); assertThat(streamsBuilderFactoryBean.isRunning()).isTrue(); }
return new KafkaStreams(builder.build(), new StreamsConfig(config));
/** * Create a {@code KafkaStreams} instance. * <p> * Note: even if you never call {@link #start()} on a {@code KafkaStreams} instance, * you still must {@link #close()} it to avoid resource leaks. * * @param topology the topology specifying the computational logic * @param props properties for {@link StreamsConfig} * @param clientSupplier the Kafka clients supplier which provides underlying producer and consumer clients * for the new {@code KafkaStreams} instance * @throws StreamsException if any fatal error occurs */ public KafkaStreams(final Topology topology, final Properties props, final KafkaClientSupplier clientSupplier) { this(topology.internalTopologyBuilder, new StreamsConfig(props), clientSupplier, Time.SYSTEM); }
/** * Create a {@code KafkaStreams} instance. * <p> * Note: even if you never call {@link #start()} on a {@code KafkaStreams} instance, * you still must {@link #close()} it to avoid resource leaks. * * @param topology the topology specifying the computational logic * @param props properties for {@link StreamsConfig} * @param clientSupplier the Kafka clients supplier which provides underlying producer and consumer clients * for the new {@code KafkaStreams} instance * @param time {@code Time} implementation; cannot be null * @throws StreamsException if any fatal error occurs */ public KafkaStreams(final Topology topology, final Properties props, final KafkaClientSupplier clientSupplier, final Time time) { this(topology.internalTopologyBuilder, new StreamsConfig(props), clientSupplier, time); }
return new KafkaStreams(builder.build(), new StreamsConfig(streamsConfiguration));
@Bean(name = KafkaStreamsDefaultConfiguration.DEFAULT_STREAMS_CONFIG_BEAN_NAME) public StreamsConfig streamsConfig() { Map<String, Object> props = new HashMap<>(); props.put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId); props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); return new StreamsConfig(props); }
public KafkaStreams constructStreamsInstance() { // Validate and set bootstrap servers config if (!originalConfig.containsKey(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG)) { effectiveConfig.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, getBootstrapServers()); } // Override with original config effectiveConfig.putAll(originalConfig); return new KafkaStreams(kstreamBuilder, new StreamsConfig(effectiveConfig)); }
private StreamsConfig toStreamsConfig(Config config) { val props = new Properties(); config.entrySet().forEach(e -> props.setProperty(e.getKey(), config.getString(e.getKey()))); return new StreamsConfig(props); } }
/** * Create a {@code KafkaStreams} instance. * <p> * Note: even if you never call {@link #start()} on a {@code KafkaStreams} instance, * you still must {@link #close()} it to avoid resource leaks. * * @param topology the topology specifying the computational logic * @param props properties for {@link StreamsConfig} * @throws StreamsException if any fatal error occurs */ public KafkaStreams(final Topology topology, final Properties props) { this(topology.internalTopologyBuilder, new StreamsConfig(props), new DefaultKafkaClientSupplier()); }
/** * Create a {@code KafkaStreams} instance. * <p> * Note: even if you never call {@link #start()} on a {@code KafkaStreams} instance, * you still must {@link #close()} it to avoid resource leaks. * * @param topology the topology specifying the computational logic * @param props properties for {@link StreamsConfig} * @param time {@code Time} implementation; cannot be null * @throws StreamsException if any fatal error occurs */ public KafkaStreams(final Topology topology, final Properties props, final Time time) { this(topology.internalTopologyBuilder, new StreamsConfig(props), new DefaultKafkaClientSupplier(), time); }
@Override public KafkaStreams make(final String ryaInstance, final StreamsQuery query) throws KafkaStreamsFactoryException { requireNonNull(ryaInstance); requireNonNull(query); // Setup the Kafka Stream program. final Properties streamsProps = new Properties(); // Configure the Kafka servers that will be talked to. streamsProps.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServersConfig); // Use the Query ID as the Application ID to ensure we resume where we left off the last time this command was run. streamsProps.put(StreamsConfig.APPLICATION_ID_CONFIG, "RyaStreams-Query-" + query.getQueryId()); // Always start at the beginning of the input topic. streamsProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); // Setup the topology that processes the Query. final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance); final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, query.getQueryId()); try { final TopologyBuilder topologyBuilder = topologyFactory.build(query.getSparql(), statementsTopic, resultsTopic, new RandomUUIDFactory()); return new KafkaStreams(topologyBuilder, new StreamsConfig(streamsProps)); } catch (final MalformedQueryException | TopologyBuilderException e) { throw new KafkaStreamsFactoryException("Could not create a KafkaStreams processing topology for query " + query.getQueryId(), e); } } }
final KafkaStreams streams = new KafkaStreams(topologyBuilder, new StreamsConfig(streamsProps));
@Override public GraphAlgorithmState<Void> configure(StreamsBuilder builder, Properties streamsConfig) { ClientUtils.createTopic(solutionSetTopic, numPartitions, replicationFactor, streamsConfig); ClientUtils.createTopic(workSetTopic, numPartitions, replicationFactor, streamsConfig); computation.prepare(builder, streamsConfig); Topology topology = builder.build(); log.info("Topology description {}", topology.describe()); streams = new KafkaStreams(topology, new StreamsConfig(streamsConfig), new PregelClientSupplier()); streams.start(); return new GraphAlgorithmState<>(streams, GraphAlgorithmState.State.CREATED, 0, 0L, null); }
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); final KafkaStreams streams = new KafkaStreams(builder, new StreamsConfig(props)); streams.cleanUp(); try {