@Override public boolean equals(Object other) { if (other == null || other.getClass() != PubSubMessage.class) { return false; } PubSubMessage otherMessage = (PubSubMessage) other; return id.equals(otherMessage.getId()) && sequence == otherMessage.getSequence(); }
@Override public PubSubMessage receive() throws PubSubException { if (uncommittedMessages.size() >= maxUncommittedMessages) { log.warn("Reached limit of max uncommitted messages: {}. Waiting for commits to proceed.", maxUncommittedMessages); return null; } if (!haveMessages()) { return null; } PubSubMessage message = receivedMessages.remove(0); uncommittedMessages.put(ImmutablePair.of(message.getId(), message.getSequence()), message); return message; }
@Override public void nextTuple() { PubSubMessage message = null; try { message = subscriber.receive(); } catch (Exception e) { log.error(e.getMessage()); } if (message == null) { Utils.sleep(1); return; } String content = message.getContent(); // If no content, it's a metadata only message. Send it on the METADATA_STREAM. if (content == null) { collector.emit(METADATA_STREAM, new Values(message.getId(), message.getMetadata()), message.getId()); } else { collector.emit(QUERY_STREAM, new Values(message.getId(), message.getContent(), message.getMetadata()), message.getId()); } }
private void handleResponse(String id, Response response) { if (response == null || response.getStatusCode() != OK_200) { log.error("Handling error for id {} with response {}", id, response); responses.offer(new PubSubMessage(id, DRPCError.CANNOT_REACH_DRPC.asJSONClip())); return; } log.info("Received for id {}: {} {}", response.getStatusCode(), id, response.getStatusText()); String body = response.getResponseBody(); PubSubMessage message = PubSubMessage.fromJSON(body); log.debug("Received for id {}:\n{}", message.getId(), message.getContent()); responses.offer(message); } }
@Override public void nextTuple() { PubSubMessage message = null; try { message = subscriber.receive(); } catch (Exception e) { log.error(e.getMessage()); } if (message == null) { Utils.sleep(1); return; } String content = message.getContent(); // If no content, it's a metadata only message. Send it on the METADATA_STREAM. if (content == null) { collector.emit(METADATA_STREAM, new Values(message.getId(), message.getMetadata()), message.getId()); } else { collector.emit(QUERY_STREAM, new Values(message.getId(), message.getContent(), message.getMetadata()), message.getId()); } }
private void handleResponse(String id, Response response) { if (response == null || response.getStatusCode() != OK_200) { log.error("Handling error for id {} with response {}", id, response); responses.offer(new PubSubMessage(id, DRPCError.CANNOT_REACH_DRPC.asJSONClip())); return; } log.info("Received for id {}: {} {}", response.getStatusCode(), id, response.getStatusText()); String body = response.getResponseBody(); PubSubMessage message = PubSubMessage.fromJSON(body); log.debug("Received for id {}:\n{}", message.getId(), message.getContent()); responses.offer(message); } }
@Override public void send(PubSubMessage message) throws PubSubException { String url = urls.get(); String id = message.getId(); String json = message.asJSON(); log.info("Posting to {} for id {}", url, id); log.debug("Posting to {} with body {}", url, json); client.preparePost(url).setBody(json).execute().toCompletableFuture() .exceptionally(this::handleException) .thenAcceptAsync(createResponseConsumer(id)); }
@Override public List<PubSubMessage> getMessages() throws PubSubException { // Try and read from DRPC. The DRPCSpout does a sleep for 1 ms if there are no tuples, so we don't have to do it. spout.nextTuple(); if (!collector.haveOutput()) { return null; } // The DRPCSpout only should have emitted one tuple List<List<Object>> tuples = collector.reset(); log.debug("Have a message through DRPC {}", tuples); List<Object> tupleAndID = tuples.get(0); // The first object is the actual DRPCSpout tuple and the second is the DRPC messageID. List<Object> tuple = (List<Object>) tupleAndID.get(0); Object drpcID = tupleAndID.get(1); // The first object in the tuple is our PubSubMessage as JSON String pubSubMessageJSON = (String) tuple.get(0); // The second object in the tuple is the serialized returnInfo added by the DRPCSpout String returnInfo = (String) tuple.get(1); log.debug("Read message\n{}\nfrom DRPC with return information {}", pubSubMessageJSON, returnInfo); PubSubMessage pubSubMessage = PubSubMessage.fromJSON(pubSubMessageJSON); // Add returnInfo as metadata. Cannot add it to pubSubMessage String id = pubSubMessage.getId(); String content = pubSubMessage.getContent(); int sequence = pubSubMessage.getSequence(); PubSubMessage message = new PubSubMessage(id, content, new Metadata(null, returnInfo), sequence); emittedIDs.put(ImmutablePair.of(id, sequence), drpcID); return Collections.singletonList(message); }
@Override public void send(PubSubMessage message) throws PubSubException { String url = urls.get(); String id = message.getId(); String json = message.asJSON(); log.info("Posting to {} for id {}", url, id); log.debug("Posting to {} with body {}", url, json); client.preparePost(url).setBody(json).execute().toCompletableFuture() .exceptionally(this::handleException) .thenAcceptAsync(createResponseConsumer(id)); }
@Override public List<PubSubMessage> getMessages() throws PubSubException { // Try and read from DRPC. The DRPCSpout does a sleep for 1 ms if there are no tuples, so we don't have to do it. spout.nextTuple(); if (!collector.haveOutput()) { return null; } // The DRPCSpout only should have emitted one tuple List<List<Object>> tuples = collector.reset(); log.debug("Have a message through DRPC {}", tuples); List<Object> tupleAndID = tuples.get(0); // The first object is the actual DRPCSpout tuple and the second is the DRPC messageID. List<Object> tuple = (List<Object>) tupleAndID.get(0); Object drpcID = tupleAndID.get(1); // The first object in the tuple is our PubSubMessage as JSON String pubSubMessageJSON = (String) tuple.get(0); // The second object in the tuple is the serialized returnInfo added by the DRPCSpout String returnInfo = (String) tuple.get(1); log.debug("Read message\n{}\nfrom DRPC with return information {}", pubSubMessageJSON, returnInfo); PubSubMessage pubSubMessage = PubSubMessage.fromJSON(pubSubMessageJSON); // Add returnInfo as metadata. Cannot add it to pubSubMessage String id = pubSubMessage.getId(); String content = pubSubMessage.getContent(); int sequence = pubSubMessage.getSequence(); PubSubMessage message = new PubSubMessage(id, content, new Metadata(null, returnInfo), sequence); emittedIDs.put(ImmutablePair.of(id, sequence), drpcID); return Collections.singletonList(message); }
@Test public void testMessagesAreLooped() { List<PubSubMessage> expected = asList(new PubSubMessage("42", Metadata.Signal.KILL), new PubSubMessage("43", Metadata.Signal.COMPLETE), new PubSubMessage("44", (Metadata.Signal) null)); List<Tuple> tuples = new ArrayList<>(); expected.forEach(m -> tuples.add(makeTuple(m.getId(), m.getMetadata()))); for (int i = 0; i < tuples.size(); i++) { bolt.execute(tuples.get(i)); Assert.assertEquals(publisher.getSent().get(i).getId(), expected.get(i).getId()); Assert.assertEquals(publisher.getSent().get(i).getMetadata(), expected.get(i).getMetadata()); Assert.assertTrue(collector.wasNthAcked(tuples.get(i), i + 1)); Assert.assertEquals(collector.getAckedCount(), i + 1); } }
@Test(timeOut = 5000L) public void testException() throws Exception { // This will hit a non-existent url and fail, testing our exceptions. Our connect and retry is low so even if // block the full amount, it's still fast. pubscriber.send(new PubSubMessage("foo", "bar")); PubSubMessage actual = fetchAsync().get(); Assert.assertNotNull(actual); Assert.assertEquals(actual.getId(), "foo"); Assert.assertEquals(actual.getContent(), CANNOT_REACH_DRPC.asJSONClip()); } }
@Test public void testExecuteMessagesAreSent() { List<PubSubMessage> expected = asList(new PubSubMessage("42", "This is a PubSubMessage", new Metadata()), new PubSubMessage("43", "This is also a PubSubMessage", new Metadata()), new PubSubMessage("44", "This is still a PubSubMessage", new Metadata())); List<Tuple> tuples = new ArrayList<>(); expected.forEach(m -> tuples.add(makeTuple(m.getId(), m.getContent(), m.getMetadata()))); for (int i = 0; i < tuples.size(); i++) { bolt.execute(tuples.get(i)); Assert.assertEquals(publisher.getSent().get(i).getId(), expected.get(i).getId()); Assert.assertEquals(publisher.getSent().get(i).getContent(), expected.get(i).getContent()); Assert.assertEquals(publisher.getSent().get(i).getMetadata(), expected.get(i).getMetadata()); Assert.assertTrue(collector.wasNthAcked(tuples.get(i), i + 1)); Assert.assertEquals(collector.getAckedCount(), i + 1); } }
@Test(timeOut = 5000L) public void testReadingNullResponse() throws Exception { CompletableFuture<Response> response = getOkFuture(null); AsyncHttpClient mockClient = mockClientWith(response); pubscriber.setClient(mockClient); pubscriber.send(new PubSubMessage("foo", "bar")); // This is async (but practically still very fast since we mocked the response), so need a timeout. PubSubMessage actual = fetchAsync().get(); Assert.assertNotNull(actual); Assert.assertEquals(actual.getId(), "foo"); Assert.assertEquals(actual.getContent(), CANNOT_REACH_DRPC.asJSONClip()); }
@Test(timeOut = 5000L) public void testReadingOkResponse() throws Exception { PubSubMessage expected = new PubSubMessage("foo", "response"); CompletableFuture<Response> response = getOkFuture(getOkResponse(expected.asJSON())); AsyncHttpClient mockClient = mockClientWith(response); pubscriber.setClient(mockClient); pubscriber.send(new PubSubMessage("foo", "bar")); // This is async (but practically still very fast since we mocked the response), so need a timeout. PubSubMessage actual = fetchAsync().get(); Assert.assertNotNull(actual); Assert.assertEquals(actual.getId(), expected.getId()); Assert.assertEquals(actual.getContent(), expected.getContent()); }
@Test(timeOut = 5000L) public void testReadingNotOkResponse() throws Exception { CompletableFuture<Response> response = getOkFuture(getNotOkResponse(500)); AsyncHttpClient mockClient = mockClientWith(response); pubscriber.setClient(mockClient); pubscriber.send(new PubSubMessage("foo", "bar")); // This is async (but practically still very fast since we mocked the response), so need a timeout. PubSubMessage actual = fetchAsync().get(); Assert.assertNotNull(actual); Assert.assertEquals(actual.getId(), "foo"); Assert.assertEquals(actual.getContent(), CANNOT_REACH_DRPC.asJSONClip()); }
@Override public void send(PubSubMessage message) throws PubSubException { Metadata metadata = message.getMetadata(); // Remove the content String content = metadata.getContent().toString(); log.debug("Removing metadata {} for result {}@{}: {}", content, message.getId(), message.getSequence(), message.getContent()); metadata.setContent(null); String serializedMessage = message.asJSON(); Tuple tuple = new DRPCTuple(new Values(serializedMessage, content)); // This sends the message through DRPC and not to the collector but it acks or fails accordingly. bolt.execute(tuple); if (!collector.isAcked()) { throw new PubSubException("Message not acked. Unable to send message through DRPC:\n " + serializedMessage); } // Otherwise, we're good to proceed collector.reset(); }
@Override public void send(PubSubMessage message) throws PubSubException { Metadata metadata = message.getMetadata(); // Remove the content String content = metadata.getContent().toString(); log.debug("Removing metadata {} for result {}@{}: {}", content, message.getId(), message.getSequence(), message.getContent()); metadata.setContent(null); String serializedMessage = message.asJSON(); Tuple tuple = new DRPCTuple(new Values(serializedMessage, content)); // This sends the message through DRPC and not to the collector but it acks or fails accordingly. bolt.execute(tuple); if (!collector.isAcked()) { throw new PubSubException("Message not acked. Unable to send message through DRPC:\n " + serializedMessage); } // Otherwise, we're good to proceed collector.reset(); }
@Test public void testNextTupleMessagesAreReceivedAndTupleIsEmitted() { // Add messages to be received from subscriber PubSubMessage messageA = new PubSubMessage("42", "This is a PubSubMessage", new Metadata()); PubSubMessage messageB = new PubSubMessage("43", "This is also a PubSubMessage", new Metadata()); subscriber.addMessages(messageA, messageB); Assert.assertEquals(subscriber.getReceived().size(), 0); Assert.assertEquals(emitter.getEmitted().size(), 0); // subscriber.receive() -> messageA spout.nextTuple(); Assert.assertEquals(subscriber.getReceived().size(), 1); Assert.assertEquals(subscriber.getReceived().get(0), messageA); Tuple emittedFirst = TupleUtils.makeTuple(TupleClassifier.Type.QUERY_TUPLE, messageA.getId(), messageA.getContent(), messageA.getMetadata()); Assert.assertEquals(emitter.getEmitted().size(), 1); Assert.assertTrue(emitter.wasNthEmitted(emittedFirst, 1)); // subscriber.receive() -> messageB spout.nextTuple(); Assert.assertEquals(subscriber.getReceived().size(), 2); Assert.assertEquals(subscriber.getReceived().get(0), messageA); Assert.assertEquals(subscriber.getReceived().get(1), messageB); Tuple emittedSecond = TupleUtils.makeTuple(TupleClassifier.Type.QUERY_TUPLE, messageB.getId(), messageB.getContent(), messageB.getMetadata()); Assert.assertEquals(emitter.getEmitted().size(), 2); Assert.assertTrue(emitter.wasNthEmitted(emittedFirst, 1)); Assert.assertTrue(emitter.wasNthEmitted(emittedSecond, 2)); }
@Test public void testSignalOnlyMessagesAreSentOnTheMetadataStream() { // Add messages to be received from subscriber PubSubMessage messageA = new PubSubMessage("42", Metadata.Signal.KILL); PubSubMessage messageB = new PubSubMessage("43", Metadata.Signal.COMPLETE); PubSubMessage messageC = new PubSubMessage("44", null, new Metadata()); subscriber.addMessages(messageA, messageB, messageC); Assert.assertEquals(subscriber.getReceived().size(), 0); Assert.assertEquals(emitter.getEmitted().size(), 0); spout.nextTuple(); spout.nextTuple(); spout.nextTuple(); Assert.assertEquals(subscriber.getReceived().size(), 3); Assert.assertEquals(subscriber.getReceived().get(0), messageA); Assert.assertEquals(subscriber.getReceived().get(1), messageB); Assert.assertEquals(subscriber.getReceived().get(2), messageC); Assert.assertEquals(emitter.getEmitted().size(), 3); Tuple emittedFirst = TupleUtils.makeTuple(TupleClassifier.Type.METADATA_TUPLE, messageA.getId(), messageA.getMetadata()); Tuple emittedSecond = TupleUtils.makeTuple(TupleClassifier.Type.METADATA_TUPLE, messageB.getId(), messageB.getMetadata()); Tuple emittedThird = TupleUtils.makeTuple(TupleClassifier.Type.METADATA_TUPLE, messageC.getId(), messageC.getMetadata()); Assert.assertTrue(emitter.wasTupleEmittedTo(emittedFirst, TopologyConstants.METADATA_STREAM)); Assert.assertTrue(emitter.wasTupleEmittedTo(emittedSecond, TopologyConstants.METADATA_STREAM)); Assert.assertTrue(emitter.wasTupleEmittedTo(emittedThird, TopologyConstants.METADATA_STREAM)); Assert.assertTrue(emitter.wasNthEmitted(emittedFirst, 1)); Assert.assertTrue(emitter.wasNthEmitted(emittedSecond, 2)); Assert.assertTrue(emitter.wasNthEmitted(emittedThird, 3)); }