private Double calculateAvg(Number sum, String field) { Number count = metrics.get(new GroupOperation(COUNT_FIELD, field, null)); if (sum == null || count == null) { return null; } return sum.doubleValue() / count.longValue(); }
@SuppressWarnings("unchecked") private static GroupOperation makeOperation(Object object) { try { Map<String, String> data = (Map<String, String>) object; String type = data.get(OPERATION_TYPE); Optional<GroupOperationType> operation = SUPPORTED_GROUP_OPERATIONS.stream().filter(t -> t.isMe(type)).findFirst(); // May or may not be present String field = data.get(OPERATION_FIELD); // May or may not be present String newName = data.get(OPERATION_NEW_NAME); // Unknown GroupOperations are ignored. return operation.isPresent() ? new GroupOperation(operation.get(), field, newName) : null; } catch (ClassCastException | NullPointerException e) { return null; } } }
/** * Creates a {@link Map} of {@link GroupOperation} to their numeric metric values from a {@link Set} of * {@link GroupOperation}. * * @param operations A set of operations. * @return A empty map of metrics that represent these operations. */ public static Map<GroupOperation, Number> makeInitialMetrics(Set<GroupOperation> operations) { Map<GroupOperation, Number> metrics = new HashMap<>(); // Initialize with nulls. for (GroupOperation operation : operations) { metrics.put(operation, null); if (operation.getType() == AVG) { // For AVG we store an addition COUNT_FIELD operation to store the count (the sum is stored in AVG) metrics.put(new GroupOperation(COUNT_FIELD, operation.getField(), null), null); } } return metrics; }
private static byte[] getGroupDataWithCount(String countField, int count) { GroupData groupData = new GroupData(new HashSet<>(singletonList(new GroupOperation(COUNT, null, countField)))); IntStream.range(0, count).forEach(i -> groupData.consume(RecordBox.get().getRecord())); return SerializerDeserializer.toBytes(groupData); }
singletonList(new GroupOperation(COUNT, null, "cnt"))); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", filterQuery, EMPTY);
setup(bolt); List<GroupOperation> operations = asList(new GroupOperation(COUNT, null, "cnt"), new GroupOperation(SUM, "fieldB", "sumB")); String queryString = makeGroupFilterQuery("ts", singletonList("1"), EQUALS, GROUP, entries, operations, Pair.of("fieldA", "A"));
singletonList(new GroupOperation(COUNT, null, "cnt"))); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", filterQuery, EMPTY);
@Test public void testCounting() { bolt = new DonableJoinBolt(config, 5, true); setup(bolt); String filterQuery = makeGroupFilterQuery("timestamp", asList("1", "2"), EQUALS, GROUP, 1, singletonList(new GroupOperation(COUNT, null, "cnt"))); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", filterQuery, EMPTY); bolt.execute(query); // Send 5 GroupData with counts 1, 2, 3, 4, 5 to the JoinBolt IntStream.range(1, 6).forEach(i -> sendRawByteTuplesTo(bolt, "42", singletonList(getGroupDataWithCount("cnt", i)))); // 1 + 2 + 3 + 4 + 5 List<BulletRecord> result = singletonList(RecordBox.get().add("cnt", 15L).getRecord()); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(result).asJSON(), COMPLETED); Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE); // Should starts buffering the query for the query tickout bolt.execute(tick); for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_POST_FINISH_BUFFER_TICKS - 1; ++i) { bolt.execute(tick); Assert.assertFalse(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); } bolt.execute(tick); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
@Test public void testGroupAllCount() { // 15 Records will be consumed bolt = ComponentUtils.prepare(new DonableFilterBolt(15, new BulletStormConfig()), collector); Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeGroupFilterQuery("timestamp", asList("1", "2"), EQUALS, GROUP, 1, singletonList(new GroupOperation(COUNT, null, "cnt"))), METADATA); bolt.execute(query); BulletRecord record = RecordBox.get().add("timestamp", "1").getRecord(); Tuple matching = makeRecordTuple(record); IntStream.range(0, 10).forEach(i -> bolt.execute(matching)); BulletRecord another = RecordBox.get().getRecord(); Tuple nonMatching = makeRecordTuple(another); IntStream.range(0, 5).forEach(i -> bolt.execute(nonMatching)); bolt.execute(nonMatching); // Two to flush bolt Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE); bolt.execute(tick); bolt.execute(tick); Assert.assertEquals(collector.getEmittedCount(), 1); GroupData actual = SerializerDeserializer.fromBytes(getRawPayloadOfNthTuple(1)); BulletRecord expected = RecordBox.get().add("cnt", 10L).getRecord(); Assert.assertTrue(isEqual(actual, expected)); }