How to use

Clip

in

com.yahoo.bullet.result

  /**
   * Write this error as a JSON Bullet error response in the {@link Meta} of a {@link Clip}.
   *
   * @return A String JSON version of this error.
   */
  public String asJSONClip() {
    return Clip.of(Meta.of(this)).asJSON();
  }
}

@Override
public Clip getResult() {
  // This has already called aggregation.getMetadata
  Clip clip = Clip.of(getMetadata());
  clip.add(aggregation.getRecords());
  return clip;
}

/**
 * Adds all the {@link BulletRecord} and the {@link Meta} from the given {@link Clip} to this.
 *
 * @param clip The clip to add.
 * @return This Clip for chaining.
 */
public Clip add(Clip clip) {
  if (clip != null) {
    add(clip.getMeta());
    add(clip.getRecords());
  }
  return this;
}

/**
 * Construct a Clip with the given {@link BulletRecord}.
 *
 * @param record The input record.
 * @return The created Clip.
 */
public static Clip of(BulletRecord record) {
  return new Clip().add(record);
}

@Test
public void testRateLimitingOnCombine() {
  RateLimitError rateLimitError = new RateLimitError(42.0, 5.0);
  bolt = new RateLimitedJoinBolt(2, rateLimitError, config);
  setup(bolt);
  Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 10));
  bolt.execute(query);
  // After consuming the 3rd one, it is rate limited and the fourth is not consumed
  List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", 4);
  Assert.assertEquals(collector.getEmittedCount(), 2);
  Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42",
                     Clip.of(sent.subList(0, 3)).add(rateLimitError.makeMeta()).asJSON(),
                     new Metadata(Metadata.Signal.FAIL, null));
  Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
  Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null));
  Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
}

/**
 * Gets the result from the data presented to the sketch as a {@link Clip}. Also adds {@link Meta} if
 * asked for.
 *
 * @param metaKey If set to a non-null value, Sketch metadata will be added to the result.
 * @param conceptKeys If provided, these {@link Concept} names will be added to the metadata.
 * @return A {@link Clip} of the results.
 */
public Clip getResult(String metaKey, Map<String, String> conceptKeys) {
  // Subclasses are charge of adding data. We'll just create an empty Clip with the metadata.
  return Clip.of(getMetadata(metaKey, conceptKeys));
}

/**
 * Returns the {@link List} of {@link BulletRecord} result so far. See {@link #getResult()} for the full result
 * with metadata.
 *
 * @return The records that are part of the result.
 */
@Override
public List<BulletRecord> getRecords() {
  try {
    incrementRate();
    Clip result = new Clip();
    result.add(window.getRecords());
    result = postAggregate(result);
    return result.getRecords();
  } catch (RuntimeException e) {
    log.error("Unable to get serialized result for query {}", this);
    return null;
  }
}

@Override
public Clip getResult(String metaKey, Map<String, String> conceptKeys) {
  Clip data = super.getResult(metaKey, conceptKeys);
  data.add(getRecords());
  return data;
}

/**
 * Terminate the query and return the final result.
 *
 * @return The final non-null {@link Clip} representing the final result.
 */
public Clip finish() {
  Clip result = getResult();
  addFinishTime(result.getMeta());
  return result;
}

private void emitResult(String id, Metadata metadata, Clip result) {
  // Metadata should not be checked. It could be null.
  collector.emit(RESULT_STREAM, new Values(id, result.asJSON(), metadata));
}

@Override
public Clip getResult() {
  Clip result = super.getResult();
  result.getRecords().forEach(this::splitFields);
  return result;
}

@Test
public void testQueryIdentifierMetadata() {
  config = configWithRawMaxAndEmptyMeta();
  enableMetadataInConfig(config, Concept.QUERY_METADATA.getName(), "meta");
  enableMetadataInConfig(config, Concept.QUERY_ID.getName(), "id");
  setup(new JoinBolt(config));
  Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY);
  bolt.execute(query);
  List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42");
  Meta meta = new Meta();
  meta.add("meta", singletonMap("id", "42"));
  Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).add(meta).asJSON(), COMPLETED);
  Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
  Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
  Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
  Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
  Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}

/**
 * Gets the aggregated records so far since the last call to {@link #reset()}. The records have a size that is at
 * most the maximum specified by the {@link Aggregation}.
 *
 * @return a {@link Clip} of the records so far.
 */
@Override
public Clip getResult() {
  return Clip.of(getRecords());
}

/**
 * Construct a Clip with the given List of {@link BulletRecord}.
 *
 * @param records The input records.
 * @return The created Clip.
 */
public static Clip of(List<BulletRecord> records) {
  return new Clip().add(records);
}

@Override
public Clip getResult(String metaKey, Map<String, String> conceptKeys) {
  merge();
  Clip result = super.getResult(metaKey, conceptKeys);
  result.add(getRecords());
  return result;
}

private void emitRateLimitError(String id, Querier querier, RateLimitError error) {
  Metadata metadata = bufferedMetadata.get(id);
  Meta meta = error.makeMeta();
  Clip clip = querier.finish();
  clip.getMeta().merge(meta);
  emitResult(id, withSignal(metadata, Metadata.Signal.FAIL), clip);
  emitMetaSignal(id, Metadata.Signal.KILL);
  updateCount(rateExceededQueries, 1L);
  removeQuery(id);
}

private void emitResult(String id, Metadata metadata, Clip result) {
  // Metadata should not be checked. It could be null.
  collector.emit(RESULT_STREAM, new Values(id, result.asJSON(), metadata));
}

private Clip postAggregate(Clip clip) {
  if (postStrategies == null) {
    return clip;
  }
  for (PostStrategy postStrategy : postStrategies) {
    clip = postStrategy.execute(clip);
  }
  for (String field : transientFields.keySet()) {
    clip.getRecords().forEach(record -> record.remove(field));
  }
  return clip;
}

  /**
   * Write this error as a JSON Bullet error response in the {@link Meta} of a {@link Clip}.
   *
   * @return A String JSON version of this error.
   */
  public String asJSONClip() {
    return Clip.of(Meta.of(this)).asJSON();
  }
}

@Test
public void testRateLimitErrorFromUpstream() {
  config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true);
  config.validate();
  setup(bolt);
  Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY);
  bolt.execute(query);
  List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1);
  Assert.assertEquals(collector.getEmittedCount(), 0);
  Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1));
  RateLimitError rateLimitError = new RateLimitError(2000.0, 1000.0);
  Tuple error = TupleUtils.makeIDTuple(TupleClassifier.Type.ERROR_TUPLE, "42", rateLimitError);
  bolt.execute(error);
  Assert.assertEquals(collector.getEmittedCount(), 2);
  Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0));
  Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42",
                     Clip.of(sent).add(rateLimitError.makeMeta()).asJSON(),
                     new Metadata(Metadata.Signal.FAIL, null));
  Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
  Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null));
  Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
  Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
  Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}