@Override public KeyValue<String, Prediction> apply(String key, JsonNode value) { //Nasty hack until we get correct method to reduce and send non or final per second aggregations to influxdb Random r = new Random(); Prediction pred = new Prediction(); pred.parse(value); String ikey = pred.consumer+"_"+pred.variation+"_"+pred.model+"_"+pred.predictedClass+"_"+pred.time+"_"+r.nextInt(); return new KeyValue<String,Prediction>(ikey,pred); }
@Override public KeyValue<K, V> transform(K key, V value) { return KeyValue.pair(key, value); } });
new KeyValue<>("alice", "asia"), new KeyValue<>("bob", "europe"), new KeyValue<>("alice", "europe"), new KeyValue<>("bob", "asia") ); new KeyValue<>("europe", 1L), // in the end, Alice is in europe new KeyValue<>("asia", 1L) // in the end, Bob is in asia ); .groupBy((userId, region) -> KeyValue.pair(region, region)) .count();
private IAnswer<Integer> rows(final Object... rows) { return () -> { final Collection<KeyValue<String, GenericRow>> output = drainCapture.getValue(); Arrays.stream(rows) .map(ImmutableList::of) .map(GenericRow::new) .forEach(row -> output.add(new KeyValue<>("no used", row))); return rows.length; }; }
@Override public KeyValue<K, V> transform(K key, V value) { action.apply(key, value); return KeyValue.pair(key, value); } });
writtenRows.add(value); rowQueue.put(new KeyValue<>(key, value));
@Override public KeyValue<byte[], String> transform(final byte[] recordKey, final String recordValue) { // The record value contains the IP address in string representation. // The original record key is ignored because we don't need it for this logic. String anonymizedIpAddress = anonymizeIpAddress(recordValue); return KeyValue.pair(recordKey, anonymizedIpAddress); }
@Override public KeyValue<String, GenericRecord> apply(final String dummy, final GenericRecord record) { return new KeyValue<>(record.get("user").toString(), record); } });
@Override public KeyValue<String, Long> transform(byte[] key, String value) { // For simplification (and unlike the traditional wordcount) we assume that the value is // a single word, i.e. we don't split the value by whitespace into potentially one or more // words. Optional<Long> count = Optional.ofNullable(stateStore.get(value)); Long incrementedCount = count.orElse(0L) + 1; stateStore.put(value, incrementedCount); return KeyValue.pair(value, incrementedCount); }
public static void main(final String[] args) { final int quantityUnderpants = args.length > 0 ? Integer.valueOf(args[0]) : 20; final int quantityJumpers = args.length > 1 ? Integer.valueOf(args[1]) : 10; final String bootstrapServers = args.length > 2 ? args[2] : "localhost:9092"; // Send Inventory final List<KeyValue<Product, Integer>> inventory = asList( new KeyValue<>(UNDERPANTS, quantityUnderpants), new KeyValue<>(JUMPERS, quantityJumpers) ); System.out.printf("Send inventory to %s%n", Topics.WAREHOUSE_INVENTORY); sendInventory(inventory, Topics.WAREHOUSE_INVENTORY, bootstrapServers); }
private static <K, V> List<KeyValue<K, V>> readKeysAndValues(int numberToRead, String bootstrapServers, Deserializer<K> keyDes, Deserializer<V> valDes, String topicName) throws InterruptedException { Properties consumerConfig = new Properties(); consumerConfig.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); consumerConfig.put(ConsumerConfig.GROUP_ID_CONFIG, "Test-Reader-" + consumerCounter++); consumerConfig.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); KafkaConsumer<K, V> consumer = new KafkaConsumer<>(consumerConfig, keyDes, valDes); consumer.subscribe(singletonList(topicName)); List<KeyValue<K, V>> actualValues = new ArrayList<>(); TestUtils.waitForCondition(() -> { ConsumerRecords<K, V> records = consumer.poll(100); for (ConsumerRecord<K, V> record : records) { actualValues.add(KeyValue.pair(record.key(), record.value())); } return actualValues.size() == numberToRead; }, 20000, "Timed out reading orders."); consumer.close(); return actualValues; }
public static <V> void produceValuesSynchronously( String topic, Collection<V> records, Properties producerConfig) throws ExecutionException, InterruptedException { Collection<KeyValue<Object, V>> keyedRecords = records.stream().map(record -> new KeyValue<>(null, record)).collect(Collectors.toList()); produceKeyValuesSynchronously(topic, keyedRecords, producerConfig); }
.groupBy((userId, region) -> KeyValue.pair(region, region)) .count()
public static void sendOrders(List<Order> orders) { List<KeyValue<String, Order>> collect = orders.stream().map(o -> new KeyValue<>(o.getId(), o)) .collect(Collectors.toList()); send(Topics.ORDERS, collect); }
public KeyValue<K, V> transform(final K key, final V value) { E eventId = idExtractor.apply(key, value); if (eventId == null) { return KeyValue.pair(key, value); } else { KeyValue<K, V> output; if (isDuplicate(eventId)) { output = null; updateTimestampOfExistingEventToPreventExpiry(eventId, context.timestamp()); } else { output = KeyValue.pair(key, value); rememberNewEvent(eventId, context.timestamp()); } return output; } }
public static void sendOrderValuations(List<OrderValidation> orderValidations) { List<KeyValue<String, OrderValidation>> collect = orderValidations.stream().map(o -> new KeyValue<>(o.getOrderId(), o)) .collect(Collectors.toList()); send(Topics.ORDER_VALIDATIONS, collect); }
assertThat(firstSession.get(0), equalTo(KeyValue.pair(userId + "@" +start+"->"+start, 1L))); 1); assertThat(secondSession.get(0), equalTo(KeyValue.pair(userId + "@" + secondSessionStart + "->" + secondSessionStart, 1L))); assertThat(results, equalTo(Arrays.asList(KeyValue.pair(new Windowed<>(userId, new SessionWindow(start, start)),1L), KeyValue.pair(new Windowed<>(userId, new SessionWindow(secondSessionStart, secondSessionStart)),1L)))); assertThat(merged, equalTo(Arrays.asList(KeyValue.pair(userId + "@" +start+"->"+start, null), KeyValue.pair(userId + "@" +secondSessionStart +"->"+secondSessionStart, null), KeyValue.pair(userId + "@" +start+"->"+secondSessionStart, 3L)))); assertThat(mergedResults, equalTo(Collections.singletonList(KeyValue.pair(new Windowed<>(userId, new SessionWindow(start, secondSessionStart)), 3L))));
.map((key, value) -> new KeyValue<>(value.getUser(), value))
return Collections.singletonList(KeyValue.pair(key, 2 * value)); } catch (SerializationException e) {