@BeforeMethod public void setup() { collector = new DRPCOutputCollector(); }
@Test public void testFailing() { Assert.assertFalse(collector.haveOutput()); Assert.assertFalse(collector.isAcked()); Assert.assertFalse(collector.isFailed()); collector.fail(null); Assert.assertFalse(collector.haveOutput()); Assert.assertFalse(collector.isAcked()); Assert.assertTrue(collector.isFailed()); }
@Test public void testAcking() { Assert.assertFalse(collector.haveOutput()); Assert.assertFalse(collector.isAcked()); Assert.assertFalse(collector.isFailed()); collector.ack(null); Assert.assertFalse(collector.haveOutput()); Assert.assertTrue(collector.isAcked()); Assert.assertFalse(collector.isFailed()); }
@Test public void testingMultipleResetsWithoutEmitting() { Assert.assertFalse(collector.haveOutput()); Assert.assertFalse(collector.isAcked()); Assert.assertFalse(collector.isFailed()); Assert.assertNull(collector.emit("foo", new Values("bar", 1), "id1")); Assert.assertTrue(collector.haveOutput()); Assert.assertFalse(collector.isAcked()); Assert.assertFalse(collector.isFailed()); List<List<Object>> tuples = collector.reset(); // All other resets are null Assert.assertNull(collector.reset()); Assert.assertNull(collector.reset()); // Check that our returned list was not reset Assert.assertNotNull(tuples); Assert.assertEquals(tuples.size(), 1); List<Object> tuple = tuples.get(0); Assert.assertEquals(tuple.get(0), asList("bar", 1)); Assert.assertEquals(tuple.get(1), "id1"); Assert.assertFalse(collector.haveOutput()); Assert.assertFalse(collector.isAcked()); Assert.assertFalse(collector.isFailed()); }
@Override public void send(PubSubMessage message) throws PubSubException { Metadata metadata = message.getMetadata(); // Remove the content String content = metadata.getContent().toString(); log.debug("Removing metadata {} for result {}@{}: {}", content, message.getId(), message.getSequence(), message.getContent()); metadata.setContent(null); String serializedMessage = message.asJSON(); Tuple tuple = new DRPCTuple(new Values(serializedMessage, content)); // This sends the message through DRPC and not to the collector but it acks or fails accordingly. bolt.execute(tuple); if (!collector.isAcked()) { throw new PubSubException("Message not acked. Unable to send message through DRPC:\n " + serializedMessage); } // Otherwise, we're good to proceed collector.reset(); }
@Override public List<PubSubMessage> getMessages() throws PubSubException { // Try and read from DRPC. The DRPCSpout does a sleep for 1 ms if there are no tuples, so we don't have to do it. spout.nextTuple(); if (!collector.haveOutput()) { return null; } // The DRPCSpout only should have emitted one tuple List<List<Object>> tuples = collector.reset(); log.debug("Have a message through DRPC {}", tuples); List<Object> tupleAndID = tuples.get(0); // The first object is the actual DRPCSpout tuple and the second is the DRPC messageID. List<Object> tuple = (List<Object>) tupleAndID.get(0); Object drpcID = tupleAndID.get(1); // The first object in the tuple is our PubSubMessage as JSON String pubSubMessageJSON = (String) tuple.get(0); // The second object in the tuple is the serialized returnInfo added by the DRPCSpout String returnInfo = (String) tuple.get(1); log.debug("Read message\n{}\nfrom DRPC with return information {}", pubSubMessageJSON, returnInfo); PubSubMessage pubSubMessage = PubSubMessage.fromJSON(pubSubMessageJSON); // Add returnInfo as metadata. Cannot add it to pubSubMessage String id = pubSubMessage.getId(); String content = pubSubMessage.getContent(); int sequence = pubSubMessage.getSequence(); PubSubMessage message = new PubSubMessage(id, content, new Metadata(null, returnInfo), sequence); emittedIDs.put(ImmutablePair.of(id, sequence), drpcID); return Collections.singletonList(message); }
/** * Returns and resets the last series of tuples emitted to this collector. This method is not idempotent. It resets * this collector state if there were any tuples to emit. Further calls will not return the tuples. They return null. * * @return The {@link List} of tuples emitted since the last call to this method. */ public List<List<Object>> reset() { if (!haveOutput()) { return null; } List<List<Object>> toReturn = tuples; // Reset the processing tuples = new ArrayList<>(); acked = false; failed = false; return toReturn; }
@Test(expectedExceptions = UnsupportedOperationException.class) public void testBoltEmit() { collector.emit(null, (Collection<Tuple>) null, null); }
@Test(expectedExceptions = UnsupportedOperationException.class) public void testBoltEmitDirect() { collector.emitDirect(0, null, (Collection<Tuple>) null, null); }
@Test(expectedExceptions = UnsupportedOperationException.class) public void testGetPendingCount() { collector.getPendingCount(); } }
@Test public void testSpoutEmit() { Assert.assertFalse(collector.haveOutput()); Assert.assertFalse(collector.isAcked()); Assert.assertFalse(collector.isFailed()); Assert.assertNull(collector.emit("foo", new Values("bar", 1), "id1")); Assert.assertNull(collector.emit("bar", new Values("baz", 2), "id2")); Assert.assertTrue(collector.haveOutput()); Assert.assertFalse(collector.isAcked()); Assert.assertFalse(collector.isFailed()); List<List<Object>> tuples = collector.reset(); Assert.assertNotNull(tuples); Assert.assertEquals(tuples.size(), 2); List<Object> first = tuples.get(0); Assert.assertEquals(first.get(0), asList("bar", 1)); Assert.assertEquals(first.get(1), "id1"); List<Object> second = tuples.get(1); Assert.assertEquals(second.get(0), asList("baz", 2)); Assert.assertEquals(second.get(1), "id2"); Assert.assertFalse(collector.haveOutput()); Assert.assertFalse(collector.isAcked()); Assert.assertFalse(collector.isFailed()); }
@Override public void send(PubSubMessage message) throws PubSubException { Metadata metadata = message.getMetadata(); // Remove the content String content = metadata.getContent().toString(); log.debug("Removing metadata {} for result {}@{}: {}", content, message.getId(), message.getSequence(), message.getContent()); metadata.setContent(null); String serializedMessage = message.asJSON(); Tuple tuple = new DRPCTuple(new Values(serializedMessage, content)); // This sends the message through DRPC and not to the collector but it acks or fails accordingly. bolt.execute(tuple); if (!collector.isAcked()) { throw new PubSubException("Message not acked. Unable to send message through DRPC:\n " + serializedMessage); } // Otherwise, we're good to proceed collector.reset(); }
@Override public List<PubSubMessage> getMessages() throws PubSubException { // Try and read from DRPC. The DRPCSpout does a sleep for 1 ms if there are no tuples, so we don't have to do it. spout.nextTuple(); if (!collector.haveOutput()) { return null; } // The DRPCSpout only should have emitted one tuple List<List<Object>> tuples = collector.reset(); log.debug("Have a message through DRPC {}", tuples); List<Object> tupleAndID = tuples.get(0); // The first object is the actual DRPCSpout tuple and the second is the DRPC messageID. List<Object> tuple = (List<Object>) tupleAndID.get(0); Object drpcID = tupleAndID.get(1); // The first object in the tuple is our PubSubMessage as JSON String pubSubMessageJSON = (String) tuple.get(0); // The second object in the tuple is the serialized returnInfo added by the DRPCSpout String returnInfo = (String) tuple.get(1); log.debug("Read message\n{}\nfrom DRPC with return information {}", pubSubMessageJSON, returnInfo); PubSubMessage pubSubMessage = PubSubMessage.fromJSON(pubSubMessageJSON); // Add returnInfo as metadata. Cannot add it to pubSubMessage String id = pubSubMessage.getId(); String content = pubSubMessage.getContent(); int sequence = pubSubMessage.getSequence(); PubSubMessage message = new PubSubMessage(id, content, new Metadata(null, returnInfo), sequence); emittedIDs.put(ImmutablePair.of(id, sequence), drpcID); return Collections.singletonList(message); }
/** * Returns and resets the last series of tuples emitted to this collector. This method is not idempotent. It resets * this collector state if there were any tuples to emit. Further calls will not return the tuples. They return null. * * @return The {@link List} of tuples emitted since the last call to this method. */ public List<List<Object>> reset() { if (!haveOutput()) { return null; } List<List<Object>> toReturn = tuples; // Reset the processing tuples = new ArrayList<>(); acked = false; failed = false; return toReturn; }
@Override public void nextTuple() { Object id = messageIDs.poll(); List<Object> tuple = tuples.poll(); if (id != null) { collector.emit(null, tuple, id); } }
@Test(expectedExceptions = UnsupportedOperationException.class) public void testSpoutEmitDirect() { collector.emitDirect(0, null, (List<Object>) null, null); }
@Test public void testSending() throws Exception { Assert.assertEquals(injectedMockBolt.getCount(), 0); PubSubMessage message = new PubSubMessage("foo", "{}", new Metadata(null, makeReturnInfo("a", "testHost", 80))); publisher.send(message); Assert.assertEquals(injectedMockBolt.getCount(), 1); Assert.assertTrue(collector.isAcked()); Assert.assertFalse(collector.isFailed()); // Output is no longer present Assert.assertFalse(collector.haveOutput()); Assert.assertNull(collector.reset()); message = new PubSubMessage("bar", "{}", new Metadata(null, makeReturnInfo("b", "testHost", 80))); publisher.send(message); Assert.assertEquals(injectedMockBolt.getCount(), 2); Assert.assertTrue(collector.isAcked()); Assert.assertFalse(collector.isFailed()); Assert.assertFalse(collector.haveOutput()); Assert.assertNull(collector.reset()); }
/** * Creates and initializes a Publisher that writes to the DRPC servers. Intended to be used inside a Storm * bolt in a Storm topology. * * @param config Needs the Storm configuration {@link Map} in {@link com.yahoo.bullet.storm.BulletStormConfig#STORM_CONFIG}. */ public DRPCResultPublisher(BulletConfig config) { // Get the Storm Config that has all the relevant cluster settings and properties Map stormConfig = config.getRequiredConfigAs(DRPCConfig.STORM_CONFIG, Map.class); collector = new DRPCOutputCollector(); // Wrap the collector in a OutputCollector (it just delegates to the underlying DRPCOutputCollector) OutputCollector boltOutputCollector = new OutputCollector(collector); bolt = new ReturnResults(); // No need for a TopologyContext bolt.prepare(stormConfig, null, boltOutputCollector); }
/** * Creates and initializes a Publisher that writes to the DRPC servers. Intended to be used inside a Storm * bolt in a Storm topology. * * @param config Needs the Storm configuration {@link Map} in {@link com.yahoo.bullet.storm.BulletStormConfig#STORM_CONFIG}. */ public DRPCResultPublisher(BulletConfig config) { // Get the Storm Config that has all the relevant cluster settings and properties Map stormConfig = config.getRequiredConfigAs(DRPCConfig.STORM_CONFIG, Map.class); collector = new DRPCOutputCollector(); // Wrap the collector in a OutputCollector (it just delegates to the underlying DRPCOutputCollector) OutputCollector boltOutputCollector = new OutputCollector(collector); bolt = new ReturnResults(); // No need for a TopologyContext bolt.prepare(stormConfig, null, boltOutputCollector); }