/** * Construct a Clip with the given List of {@link BulletRecord}. * * @param records The input records. * @return The created Clip. */ public static Clip of(List<BulletRecord> records) { return new Clip().add(records); }
/** * Construct a Clip with the given metadata. * * @param meta The Meta to add. The objects in the Meta must be serializable to JSON * with {@link com.google.gson.Gson}. * @return This object for chaining */ public static Clip of(Meta meta) { return new Clip().add(meta); } }
/** * Construct a Clip with the given {@link BulletRecord}. * * @param record The input record. * @return The created Clip. */ public static Clip of(BulletRecord record) { return new Clip().add(record); }
@Override public Clip getResult(String metaKey, Map<String, String> conceptKeys) { Clip data = super.getResult(metaKey, conceptKeys); data.add(getRecords()); return data; }
@Override public Clip getResult(String metaKey, Map<String, String> conceptKeys) { merge(); Clip data = super.getResult(metaKey, conceptKeys); return data.add(getCount()); }
@Override public Clip getResult(String metaKey, Map<String, String> conceptKeys) { merge(); Clip data = super.getResult(metaKey, conceptKeys); data.add(getRecords()); return data; }
@Override public Clip getResult(String metaKey, Map<String, String> conceptKeys) { merge(); Clip result = super.getResult(metaKey, conceptKeys); result.add(getRecords()); return result; }
@Override public Clip getResult() { // This has already called aggregation.getMetadata Clip clip = Clip.of(getMetadata()); clip.add(aggregation.getRecords()); return clip; }
/** * Returns the {@link List} of {@link BulletRecord} result so far. See {@link #getResult()} for the full result * with metadata. * * @return The records that are part of the result. */ @Override public List<BulletRecord> getRecords() { try { incrementRate(); Clip result = new Clip(); result.add(window.getRecords()); result = postAggregate(result); return result.getRecords(); } catch (RuntimeException e) { log.error("Unable to get serialized result for query {}", this); return null; } }
/** * Gets the resulting {@link Clip} of the results so far. * * @return A non-null {@link Clip} representing the aggregated result. */ @Override public Clip getResult() { Clip result; try { incrementRate(); result = window.getResult(); result = postAggregate(result); result.add(getResultMetadata()); } catch (RuntimeException e) { log.error("Unable to get serialized data for query {}", this); result = Clip.of(getErrorMeta(e)); } return result; }
@Test public void testRateLimitErrorFromUpstream() { config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); config.validate(); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1); Assert.assertEquals(collector.getEmittedCount(), 0); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1)); RateLimitError rateLimitError = new RateLimitError(2000.0, 1000.0); Tuple error = TupleUtils.makeIDTuple(TupleClassifier.Type.ERROR_TUPLE, "42", rateLimitError); bolt.execute(error); Assert.assertEquals(collector.getEmittedCount(), 2); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0)); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).add(rateLimitError.makeMeta()).asJSON(), new Metadata(Metadata.Signal.FAIL, null)); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
@Test public void testRateLimitingOnCombine() { RateLimitError rateLimitError = new RateLimitError(42.0, 5.0); bolt = new RateLimitedJoinBolt(2, rateLimitError, config); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 10)); bolt.execute(query); // After consuming the 3rd one, it is rate limited and the fourth is not consumed List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", 4); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent.subList(0, 3)).add(rateLimitError.makeMeta()).asJSON(), new Metadata(Metadata.Signal.FAIL, null)); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); }
@Test public void testQueryIdentifierMetadata() { config = configWithRawMaxAndEmptyMeta(); enableMetadataInConfig(config, Concept.QUERY_METADATA.getName(), "meta"); enableMetadataInConfig(config, Concept.QUERY_ID.getName(), "id"); setup(new JoinBolt(config)); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42"); Meta meta = new Meta(); meta.add("meta", singletonMap("id", "42")); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).add(meta).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
@Test public void testUnknownConceptMetadata() { config = configWithRawMaxAndEmptyMeta(); enableMetadataInConfig(config, Concept.QUERY_METADATA.getName(), "meta"); enableMetadataInConfig(config, Concept.QUERY_ID.getName(), "id"); enableMetadataInConfig(config, "foo", "bar"); setup(new JoinBolt(config)); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42"); Meta meta = new Meta(); meta.add("meta", singletonMap("id", "42")); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).add(meta).asJSON(), COMPLETED); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1); Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1); }
@Test public void testRateLimitingWithTicks() { RateLimitError rateLimitError = new RateLimitError(42.0, 5.0); bolt = new RateLimitedJoinBolt(2, rateLimitError, config); setup(bolt); Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 10)); bolt.execute(query); List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", 2); Assert.assertEquals(collector.getEmittedCount(), 0); Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE); bolt.execute(tick); Assert.assertEquals(collector.getEmittedCount(), 2); Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).add(rateLimitError.makeMeta()).asJSON(), new Metadata(Metadata.Signal.FAIL, null)); Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected)); Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null)); Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata)); }