KStream<byte[], JsonNode> source = builder.stream(Serdes.ByteArray(),jsonSerde,topic); source.filter(new Predicate<byte[], JsonNode>() {
source.filter( new Predicate<String, JsonNode>()
final KTable<Integer, Integer> sumOfOddNumbers = input .filter((k, v) -> v % 2 != 0)
.filter((dummy, value) -> value.getIsNew())
.filter((dummy, value) -> value.getIsNew())
.filter((regionName, count) -> count != null);
.filter((windowedUserId, count) -> count != null) .map((windowedUserId, count) -> new KeyValue<>(windowedUserId.toString(), count));
.filter((dummy, record) -> isArticle(record))
.filter((k, v) -> v % 2 == 0) .selectKey((k, v) -> 1)
KStream<String, Order> orders = builder .stream(ORDERS.name(), serdes2) .filter((id, order) -> OrderState.CREATED.equals(order.getState())); .filter((k1, v) -> v != null) .filter((k, total) -> total >= numberOfRules) validations.filter((id, rule) -> FAIL.equals(rule.getValidationResult())) .join(orders, (id, order) ->
playEvents.filter((region, event) -> event.getDuration() >= MIN_CHARTABLE_DURATION)
@Bean public KStream<Integer, String> kStream(StreamsBuilder kStreamBuilder) { KStream<Integer, String> stream = kStreamBuilder.stream(STREAMING_TOPIC1); stream.mapValues((ValueMapper<String, String>) String::toUpperCase) .mapValues(Foo::new) .through(FOOS, Produced.with(Serdes.Integer(), new JsonSerde<Foo>() { })) .mapValues(Foo::getName) .groupByKey() .windowedBy(TimeWindows.of(1000)) .reduce((value1, value2) -> value1 + value2, Materialized.as("windowStore")) .toStream() .map((windowedId, value) -> new KeyValue<>(windowedId.key(), value)) .filter((i, s) -> s.length() > 40).to(streamingTopic2); stream.print(Printed.toSysOut()); return stream; }
KStream<String, Order> orders = builder .stream(ORDERS.name(), Consumed.with(ORDERS.keySerde(), ORDERS.valueSerde())) .filter((id, order) -> OrderState.CREATED.equals(order.getState())); .filter((k, v) -> v != null)//When elements are evicted from a session window they create delete events. Filter these out. .selectKey((id, orderValue) -> orderValue.getOrder().getId());
.filter((id, order) -> OrderState.CREATED.equals(order.getState()))
public static void main(String[] args) { // create properties Properties properties = new Properties(); properties.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:9092"); properties.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "demo-kafka-streams"); properties.setProperty(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class.getName()); properties.setProperty(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.StringSerde.class.getName()); // create a topology StreamsBuilder streamsBuilder = new StreamsBuilder(); // input topic KStream<String, String> inputTopic = streamsBuilder.stream("twitter_tweets"); KStream<String, String> filteredStream = inputTopic.filter( // filter for tweets which has a user of over 10000 followers (k, jsonTweet) -> extractUserFollowersInTweet(jsonTweet) > 10000 ); filteredStream.to("important_tweets"); // build the topology KafkaStreams kafkaStreams = new KafkaStreams( streamsBuilder.build(), properties ); // start our streams application kafkaStreams.start(); }
public static void main(String[] args) { Properties props = new Properties(); props.put(StreamsConfig.APPLICATION_ID_CONFIG, "text-transformer"); props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); props.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 5 * 1000); props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); StreamsBuilder builder = new StreamsBuilder(); builder.stream("text-input", Consumed.with(Serdes.String(), Serdes.String())) .filter((key, value) -> value.contains("a")) .mapValues(text -> text.toUpperCase()) .to("text-output", Produced.with(Serdes.String(), Serdes.String())); KafkaStreams streams = new KafkaStreams(builder.build(), props); streams.start(); } }
private static KafkaStreams createWikipediaStreamsInstance(String bootstrapServers) { final Serializer<JsonNode> jsonSerializer = new JsonSerializer(); final Deserializer<JsonNode> jsonDeserializer = new JsonDeserializer(); final Serde<JsonNode> jsonSerde = Serdes.serdeFrom(jsonSerializer, jsonDeserializer); KStreamBuilder builder = new KStreamBuilder(); Properties props = new Properties(); props.put(StreamsConfig.APPLICATION_ID_CONFIG, "wikipedia-streams"); props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); KStream<JsonNode, JsonNode> wikipediaRaw = builder.stream(jsonSerde, jsonSerde, "wikipedia-raw"); KStream<String, WikipediaMessage> wikipediaParsed = wikipediaRaw.map(WikipediaMessage::parceIRC) .filter(WikipediaMessage::filterNonNull) .through(Serdes.String(), new JsonPOJOSerde<>(WikipediaMessage.class), "wikipedia-parsed"); KTable<String, Long> totalEditsByUser = wikipediaParsed .filter((key, value) -> value.type == WikipediaMessage.Type.EDIT) .countByKey(Serdes.String(), "wikipedia-edits-by-user"); //some print totalEditsByUser.toStream().process(() -> new AbstractProcessor<String, Long>() { @Override public void process(String user, Long numEdits) { System.out.println("USER: " + user + " num.edits: " + numEdits); } }); return new KafkaStreams(builder, props); }
@StreamListener("input") @SendTo("output") public KStream<Integer, ProductStatus> process(KStream<Object, Product> input) { return input .filter((key, product) -> productIds().contains(product.getId())) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Serialized.with(new JsonSerde<>(Product.class), new JsonSerde<>(Product.class))) .windowedBy(timeWindows) .count(Materialized.as("product-counts")) .toStream() .map((key, value) -> new KeyValue<>(key.key().id, new ProductStatus(key.key().id, value, Instant.ofEpochMilli(key.window().start()).atZone(ZoneId.systemDefault()).toLocalTime(), Instant.ofEpochMilli(key.window().end()).atZone(ZoneId.systemDefault()).toLocalTime()))); }
public void stream(String bootstrapServers) throws Exception { waitForTopics(bootstrapServers); StreamsBuilder builder = new StreamsBuilder(); builder .stream(TOPICS, Consumed.with(Serdes.String(), Serdes.String())) .mapValues(cdcChange::toTelegraf) .filter((k, v) -> v != null && !v.isEmpty()) .peek((k, v) -> System.out.println(v)) .to("telegraf", Produced.with(Serdes.String(), Serdes.String())); Topology build = builder.build(); System.out.println(build.describe()); KafkaStreams kafkaStreams = new KafkaStreams(build, buildProducerProperties(bootstrapServers)); kafkaStreams.cleanUp(); kafkaStreams.start(); }
@StreamListener("input") @SendTo("output") public KStream<Integer, Long> process(KStream<Object, Product> input) { return input .filter((key, product) -> productIds().contains(product.getId())) .map((key, value) -> new KeyValue<>(value.id, value)) .groupByKey(Serialized.with(Serdes.Integer(), new JsonSerde<>(Product.class))) .count(Materialized.<Integer, Long, KeyValueStore<Bytes, byte[]>>as(STORE_NAME) .withKeySerde(Serdes.Integer()) .withValueSerde(Serdes.Long())) .toStream(); }