final KStream<Long, CustomerOrder> customerOrdersStream = ordersStream.join(customers, (orderId, order) -> order.getCustomerId(), (order, customer) -> new CustomerOrder(customer, final KStream<Long, EnrichedOrder> enrichedOrdersStream = customerOrdersStream.join(products, (orderId, customerOrder) -> customerOrder .productId(),
private KafkaStreams processStreams(final String bootstrapServers, final String stateDir) { final StreamsBuilder builder = new StreamsBuilder(); //Create the streams/tables for the join final KStream<String, Order> orders = builder.stream(ORDERS.name(), Consumed.with(ORDERS.keySerde(), ORDERS.valueSerde())); final KStream<String, Payment> payments = builder.stream(PAYMENTS.name(), Consumed.with(PAYMENTS.keySerde(), PAYMENTS.valueSerde())) //Rekey payments to be by OrderId for the windowed join .selectKey((s, payment) -> payment.getOrderId()); final GlobalKTable<Long, Customer> customers = builder.globalTable(CUSTOMERS.name(), Consumed.with(CUSTOMERS.keySerde(), CUSTOMERS.valueSerde())); final Joined<String, Order, Payment> serdes = Joined .with(ORDERS.keySerde(), ORDERS.valueSerde(), PAYMENTS.valueSerde()); //Join the two streams and the table then send an email for each orders.join(payments, EmailTuple::new, //Join Orders and Payments streams JoinWindows.of(MIN), serdes) //Next join to the GKTable of Customers .join(customers, (key1, tuple) -> tuple.order.getCustomerId(), // note how, because we use a GKtable, we can join on any attribute of the Customer. EmailTuple::setCustomer) //Now for each tuple send an email. .peek((key, emailTuple) -> emailer.sendEmail(emailTuple) ); return new KafkaStreams(builder.build(), baseStreamsConfig(bootstrapServers, stateDir, SERVICE_APP_ID)); }
.join(warehouseInventory, KeyValue::new, Joined.with(Topics.WAREHOUSE_INVENTORY.keySerde(), Topics.ORDERS.valueSerde(), Serdes.Integer()))
userPurchases.join(usersGlobalTable,
KStream<K, Tuple2<EdgeWithValue<K, EV>, VV>> edgesWithSources = edgesBySource() .join(vertices, Tuple2::new, Joined.with(keySerde(), new KryoSerde<>(), vertexValueSerde())); KTable<K, Map<EdgeWithValue<K, EV>, VV>> neighborsGroupedByTarget = edgesWithSources .map(new MapNeighbors(EdgeWithValue::target)) KStream<K, Tuple2<EdgeWithValue<K, EV>, VV>> edgesWithTargets = edgesByTarget() .join(vertices, Tuple2::new, Joined.with(keySerde(), new KryoSerde<>(), vertexValueSerde())); KTable<K, Map<EdgeWithValue<K, EV>, VV>> neighborsGroupedBySource = edgesWithTargets .map(new MapNeighbors(EdgeWithValue::source))
.join( addresses.filter((key, address) -> address != null && "USA".equals(address.country)), (member, address) -> new Aggregate().withMember(member).withAddress(address),
public KGraph<K, VV, EV> filterOnVertices(Predicate<K, VV> vertexFilter) { KTable<K, VV> filteredVertices = vertices.filter(vertexFilter); KTable<Edge<K>, EV> remainingEdges = edgesBySource() .join(filteredVertices, (e, v) -> e, Joined.with(keySerde(), new KryoSerde<>(), vertexValueSerde())) .map((k, edge) -> new KeyValue<>(edge.target(), edge)) .join(filteredVertices, (e, v) -> e, Joined.with(keySerde(), new KryoSerde<>(), vertexValueSerde())) .map((k, edge) -> new KeyValue<>(new Edge<>(edge.source(), edge.target()), edge.value())) .groupByKey(Serialized.with(new KryoSerde<>(), edgeValueSerde())) .reduce((v1, v2) -> v2, Materialized.<Edge<K>, EV, KeyValueStore<Bytes, byte[]>>as(generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(edgeValueSerde())); return new KGraph<>(filteredVertices, remainingEdges, serialized); }
KStream<K, Tuple2<EdgeWithValue<K, EV>, VV>> edgesWithSources = edgesBySource() .join(vertices, Tuple2::new, Joined.with(keySerde(), new KryoSerde<>(), vertexValueSerde())); KTable<K, Map<EdgeWithValue<K, EV>, VV>> neighborsGroupedByTarget = edgesWithSources .map(new MapNeighbors(EdgeWithValue::target)) KStream<K, Tuple2<EdgeWithValue<K, EV>, VV>> edgesWithTargets = edgesByTarget() .join(vertices, Tuple2::new, Joined.with(keySerde(), new KryoSerde<>(), vertexValueSerde())); KTable<K, Map<EdgeWithValue<K, EV>, VV>> neighborsGroupedBySource = edgesWithTargets .map(new MapNeighbors(EdgeWithValue::source))
public KGraph<K, VV, EV> subgraph(Predicate<K, VV> vertexFilter, Predicate<Edge<K>, EV> edgeFilter) { KTable<K, VV> filteredVertices = vertices.filter(vertexFilter); KTable<Edge<K>, EV> remainingEdges = edgesBySource() .join(filteredVertices, (e, v) -> e, Joined.with(keySerde(), new KryoSerde<>(), vertexValueSerde())) .map((k, edge) -> new KeyValue<>(edge.target(), edge)) .join(filteredVertices, (e, v) -> e, Joined.with(keySerde(), new KryoSerde<>(), vertexValueSerde())) .map((k, edge) -> new KeyValue<>(new Edge<>(edge.source(), edge.target()), edge.value())) .groupByKey(Serialized.with(new KryoSerde<>(), edgeValueSerde())) .reduce((v1, v2) -> v2, Materialized.with(new KryoSerde<>(), edgeValueSerde())); KTable<Edge<K>, EV> filteredEdges = remainingEdges .filter(edgeFilter, Materialized.<Edge<K>, EV, KeyValueStore<Bytes, byte[]>>as(generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(edgeValueSerde())); return new KGraph<>(filteredVertices, filteredEdges, serialized); }
sensorDataStream.join(idTable, (v1, v2) -> v1) .foreach((k,v) -> LOGGER.info("This is a valid sensor. {}", v.toString()));
private KafkaStreams processStreams(final String bootstrapServers, final String stateDir) throws IOException { final StreamsBuilder builder = new StreamsBuilder(); //Create the streams/tables for the join final KStream<String, Order> orders = builder.stream(ORDERS.name(), Consumed.with(ORDERS.keySerde(), ORDERS.valueSerde())); final GlobalKTable<String, Customer> customers = builder.globalTable(CUSTOMERS.name(), Consumed.with(CUSTOMERS.keySerde(), CUSTOMERS.valueSerde())); // // Join a stream and a table then send an email for each // GlobalKTable to stream join takes three arguments: Table, mapping of stream (key,value) to table key for join // And the join function - takes values from stream and table and returns result orders.join(customers, (orderID, order) -> order.getCustomerId(), (order, customer) -> new EmailTuple(order,customer)) //Now for each tuple send an email. .peek((key, emailTuple) -> emailer.sendEmail(emailTuple) ); return new KafkaStreams(builder.build(), configStreams(bootstrapServers, stateDir, SERVICE_APP_ID)); }