@Bean public KStream<Integer, String> kStream(StreamsBuilder kStreamBuilder) { KStream<Integer, String> stream = kStreamBuilder.stream(STREAMING_TOPIC1); stream.mapValues((ValueMapper<String, String>) String::toUpperCase) .mapValues(Foo::new) .through(FOOS, Produced.with(Serdes.Integer(), new JsonSerde<Foo>() { })) .mapValues(Foo::getName) .groupByKey() .windowedBy(TimeWindows.of(1000)) .reduce((value1, value2) -> value1 + value2, Materialized.as("windowStore")) .toStream() .map((windowedId, value) -> new KeyValue<>(windowedId.key(), value)) .filter((i, s) -> s.length() > 40).to(streamingTopic2); stream.print(Printed.toSysOut()); return stream; }
.map((dummy, record) -> new KeyValue<>(record.get("user").toString(), record)) .through("PageViewsByUser");
return new KeyValue<>(record.get("user").toString(), record); }).through("PageViewsByUser");
private static KafkaStreams createWikipediaStreamsInstance(String bootstrapServers) { final Serializer<JsonNode> jsonSerializer = new JsonSerializer(); final Deserializer<JsonNode> jsonDeserializer = new JsonDeserializer(); final Serde<JsonNode> jsonSerde = Serdes.serdeFrom(jsonSerializer, jsonDeserializer); KStreamBuilder builder = new KStreamBuilder(); Properties props = new Properties(); props.put(StreamsConfig.APPLICATION_ID_CONFIG, "wikipedia-streams"); props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); KStream<JsonNode, JsonNode> wikipediaRaw = builder.stream(jsonSerde, jsonSerde, "wikipedia-raw"); KStream<String, WikipediaMessage> wikipediaParsed = wikipediaRaw.map(WikipediaMessage::parceIRC) .filter(WikipediaMessage::filterNonNull) .through(Serdes.String(), new JsonPOJOSerde<>(WikipediaMessage.class), "wikipedia-parsed"); KTable<String, Long> totalEditsByUser = wikipediaParsed .filter((key, value) -> value.type == WikipediaMessage.Type.EDIT) .countByKey(Serdes.String(), "wikipedia-edits-by-user"); //some print totalEditsByUser.toStream().process(() -> new AbstractProcessor<String, Long>() { @Override public void process(String user, Long numEdits) { System.out.println("USER: " + user + " num.edits: " + numEdits); } }); return new KafkaStreams(builder, props); }
public static Topology topology(final String queryName, final String inputTopic, final String outputTopic) { // Build query ComplexStreamsBuilder builder = new ComplexStreamsBuilder(); CEPStream<String, StockEvent> stream = builder.stream(inputTopic); KStream<String, Sequence<String, StockEvent>> stocks = stream.query(queryName, Patterns.STOCKS); stocks.mapValues(seq -> sequenceAsJson(seq)) .through(outputTopic, Produced.with(null, Serdes.String())) .print(Printed.toSysOut()); return builder.build(); }
.filter((String userName, OrderUser orderUser) -> orderUser.userAddress != null) .map((String userName, OrderUser orderUser) -> new KeyValue<String, OrderUser>(orderUser.itemName, orderUser)) .through(Serdes.String(), SerdesFactory.serdFrom(OrderUser.class), (String key, OrderUser orderUser, int numPartitions) -> (orderUser.getItemName().hashCode() & 0x7FFFFFFF) % numPartitions, "orderuser-repartition-by-item") .leftJoin(itemTable, (OrderUser orderUser, Item item) -> OrderUserItem.fromOrderUser(orderUser, item), Serdes.String(), SerdesFactory.serdFrom(OrderUser.class)) .filter((String item, OrderUserItem orderUserItem) -> StringUtils.compare(orderUserItem.userAddress, orderUserItem.itemAddress) == 0)
public static void main(String[] args) { StreamsConfig streamingConfig = new StreamsConfig(getProperties()); JsonSerializer<StockTransactionCollector> stockTransactionsSerializer = new JsonSerializer<>(); JsonDeserializer<StockTransactionCollector> stockTransactionsDeserializer = new JsonDeserializer<>(StockTransactionCollector.class); JsonDeserializer<StockTransaction> stockTxnDeserializer = new JsonDeserializer<>(StockTransaction.class); JsonSerializer<StockTransaction> stockTxnJsonSerializer = new JsonSerializer<>(); Serde<StockTransaction> transactionSerde = Serdes.serdeFrom(stockTxnJsonSerializer,stockTxnDeserializer); StringSerializer stringSerializer = new StringSerializer(); StringDeserializer stringDeserializer = new StringDeserializer(); Serde<String> stringSerde = Serdes.serdeFrom(stringSerializer,stringDeserializer); Serde<StockTransactionCollector> collectorSerde = Serdes.serdeFrom(stockTransactionsSerializer,stockTransactionsDeserializer); WindowedSerializer<String> windowedSerializer = new WindowedSerializer<>(stringSerializer); WindowedDeserializer<String> windowedDeserializer = new WindowedDeserializer<>(stringDeserializer); Serde<Windowed<String>> windowedSerde = Serdes.serdeFrom(windowedSerializer,windowedDeserializer); KStreamBuilder kStreamBuilder = new KStreamBuilder(); KStream<String,StockTransaction> transactionKStream = kStreamBuilder.stream(stringSerde,transactionSerde,"stocks"); transactionKStream.map((k,v)-> new KeyValue<>(v.getSymbol(),v)) .through(stringSerde, transactionSerde,"stocks-out") .groupBy((k,v) -> k, stringSerde, transactionSerde) .aggregate(StockTransactionCollector::new, (k, v, stockTransactionCollector) -> stockTransactionCollector.add(v), TimeWindows.of(10000), collectorSerde, "stock-summaries") .to(windowedSerde,collectorSerde,"transaction-summary"); System.out.println("Starting StockStreams Example"); KafkaStreams kafkaStreams = new KafkaStreams(kStreamBuilder,streamingConfig); kafkaStreams.start(); System.out.println("Now started StockStreams Example"); }
.map(((key, average) -> new KeyValue<>(key.key(), new Average(average, key.window().start(), key.window().start() + 300000)))) .through(TopicNames.AVERAGE_TEMPS, Produced.with(Serdes.String(), new JsonSerde<>(Average.class))) .foreach((key, average) -> LOGGER.info(String.format("Received average %s for id %s on %s", average, key, TopicNames.AVERAGE_TEMPS)));