/** * Adds all the {@link BulletRecord} and the {@link Meta} from the given {@link Clip} to this. * * @param clip The clip to add. * @return This Clip for chaining. */ public Clip add(Clip clip) { if (clip != null) { add(clip.getMeta()); add(clip.getRecords()); } return this; }
@Test public void testRateLimitingOnCombine() { RateLimitError rateLimitError = new RateLimitError(42.0, 5.0); bolt = new RateLimitedJoinBolt(2, rateLimitError, config); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 10)); bolt.execute(query); // After consuming the 3rd one, it is rate limited and the fourth is not consumed List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", 4); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent.subList(0, 3)).add(rateLimitError.makeMeta()).asJSON(), new Metadata(Metadata.Signal.FAIL, null)); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); }
/** * Gets the result from the data presented to the sketch as a {@link Clip}. Also adds {@link Meta} if * asked for. * * @param metaKey If set to a non-null value, Sketch metadata will be added to the result. * @param conceptKeys If provided, these {@link Concept} names will be added to the metadata. * @return A {@link Clip} of the results. */ public Clip getResult(String metaKey, Map<String, String> conceptKeys) { // Subclasses are charge of adding data. We'll just create an empty Clip with the metadata. return Clip.of(getMetadata(metaKey, conceptKeys)); }
/** * Returns the {@link List} of {@link BulletRecord} result so far. See {@link #getResult()} for the full result * with metadata. * * @return The records that are part of the result. */ @Override public List<BulletRecord> getRecords() { try { incrementRate(); Clip result = new Clip(); result.add(window.getRecords()); result = postAggregate(result); return result.getRecords(); } catch (RuntimeException e) { log.error("Unable to get serialized result for query {}", this); return null; } }
@Override public Clip getResult(String metaKey, Map<String, String> conceptKeys) { Clip data = super.getResult(metaKey, conceptKeys); data.add(getRecords()); return data; }
/** * Terminate the query and return the final result. * * @return The final non-null {@link Clip} representing the final result. */ public Clip finish() { Clip result = getResult(); addFinishTime(result.getMeta()); return result; }
private void emitResult(String id, Metadata metadata, Clip result) { // Metadata should not be checked. It could be null. collector.emit(RESULT_STREAM, new Values(id, result.asJSON(), metadata)); }
@Override public Clip getResult() { Clip result = super.getResult(); result.getRecords().forEach(this::splitFields); return result; }
@Test public void testQueryIdentifierMetadata() { config = configWithRawMaxAndEmptyMeta(); enableMetadataInConfig(config, Concept.QUERY_METADATA.getName(), "meta"); enableMetadataInConfig(config, Concept.QUERY_ID.getName(), "id"); setup(new JoinBolt(config)); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42"); Meta meta = new Meta(); meta.add("meta", singletonMap("id", "42")); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).add(meta).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
/** * Gets the aggregated records so far since the last call to {@link #reset()}. The records have a size that is at * most the maximum specified by the {@link Aggregation}. * * @return a {@link Clip} of the records so far. */ @Override public Clip getResult() { return Clip.of(getRecords()); }
@Override public Clip getResult(String metaKey, Map<String, String> conceptKeys) { merge(); Clip result = super.getResult(metaKey, conceptKeys); result.add(getRecords()); return result; }
private void emitRateLimitError(String id, Querier querier, RateLimitError error) { Metadata metadata = bufferedMetadata.get(id); Meta meta = error.makeMeta(); Clip clip = querier.finish(); clip.getMeta().merge(meta); emitResult(id, withSignal(metadata, Metadata.Signal.FAIL), clip); emitMetaSignal(id, Metadata.Signal.KILL); updateCount(rateExceededQueries, 1L); removeQuery(id); }
private void emitResult(String id, Metadata metadata, Clip result) { // Metadata should not be checked. It could be null. collector.emit(RESULT_STREAM, new Values(id, result.asJSON(), metadata)); }
private Clip postAggregate(Clip clip) { if (postStrategies == null) { return clip; } for (PostStrategy postStrategy : postStrategies) { clip = postStrategy.execute(clip); } for (String field : transientFields.keySet()) { clip.getRecords().forEach(record -> record.remove(field)); } return clip; }
@Test public void testRateLimitErrorFromUpstream() { config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); config.validate(); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1)); RateLimitError rateLimitError = new RateLimitError(2000.0, 1000.0); Tuple error = TupleUtils.makeIDTuple(TupleClassifier.Type.ERROR_TUPLE, "42", rateLimitError); bolt.execute(error); Assert.assertEquals(collector.getEmittedCount(), 2); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0)); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).add(rateLimitError.makeMeta()).asJSON(), new Metadata(Metadata.Signal.FAIL, null)); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }