private void handleResponse(String id, Response response) { if (response == null || response.getStatusCode() != OK_200) { log.error("Handling error for id {} with response {}", id, response); responses.offer(new PubSubMessage(id, DRPCError.CANNOT_REACH_DRPC.asJSONClip())); return; } log.info("Received for id {}: {} {}", response.getStatusCode(), id, response.getStatusText()); String body = response.getResponseBody(); PubSubMessage message = PubSubMessage.fromJSON(body); log.debug("Received for id {}:\n{}", message.getId(), message.getContent()); responses.offer(message); } }
public static String makeMessage(String id, String content, int sequence) { return new PubSubMessage(id, content, sequence).asJSON(); } public static Object makeMessageID(String id, int index) {
@Override public void nextTuple() { PubSubMessage message = null; try { message = subscriber.receive(); } catch (Exception e) { log.error(e.getMessage()); } if (message == null) { Utils.sleep(1); return; } String content = message.getContent(); // If no content, it's a metadata only message. Send it on the METADATA_STREAM. if (content == null) { collector.emit(METADATA_STREAM, new Values(message.getId(), message.getMetadata()), message.getId()); } else { collector.emit(QUERY_STREAM, new Values(message.getId(), message.getContent(), message.getMetadata()), message.getId()); } }
@Override public boolean equals(Object other) { if (other == null || other.getClass() != PubSubMessage.class) { return false; } PubSubMessage otherMessage = (PubSubMessage) other; return id.equals(otherMessage.getId()) && sequence == otherMessage.getSequence(); }
@Override public void send(PubSubMessage message) throws PubSubException { Metadata metadata = message.getMetadata(); // Remove the content String content = metadata.getContent().toString(); log.debug("Removing metadata {} for result {}@{}: {}", content, message.getId(), message.getSequence(), message.getContent()); metadata.setContent(null); String serializedMessage = message.asJSON(); Tuple tuple = new DRPCTuple(new Values(serializedMessage, content)); // This sends the message through DRPC and not to the collector but it acks or fails accordingly. bolt.execute(tuple); if (!collector.isAcked()) { throw new PubSubException("Message not acked. Unable to send message through DRPC:\n " + serializedMessage); } // Otherwise, we're good to proceed collector.reset(); }
Assert.assertEquals(actual.getId(), "foo"); Assert.assertEquals(actual.getSequence(), 0); Assert.assertEquals(actual.getContent(), "{}"); Assert.assertFalse(actual.hasSignal()); Assert.assertTrue(actual.hasMetadata()); Metadata metadata = actual.getMetadata(); Assert.assertEquals(metadata.getContent(), makeReturnInfo("fakefoo", "testHost", 0)); Assert.assertEquals(actual.getId(), "foo"); Assert.assertEquals(actual.getSequence(), 1); Assert.assertEquals(actual.getContent(), "{'duration': 2000}"); Assert.assertFalse(actual.hasSignal()); Assert.assertTrue(actual.hasMetadata()); metadata = actual.getMetadata(); Assert.assertEquals(metadata.getContent(), makeReturnInfo("fakefoo", "testHost", 1)); Assert.assertEquals(actual.getId(), "bar"); Assert.assertEquals(actual.getSequence(), 0); Assert.assertEquals(actual.getContent(), "{}"); Assert.assertFalse(actual.hasSignal()); Assert.assertTrue(actual.hasMetadata()); metadata = actual.getMetadata(); Assert.assertEquals(metadata.getContent(), makeReturnInfo("fakebar", "testHost", 2));
@Test public void testExecuteMessagesAreSent() { List<PubSubMessage> expected = asList(new PubSubMessage("42", "This is a PubSubMessage", new Metadata()), new PubSubMessage("43", "This is also a PubSubMessage", new Metadata()), new PubSubMessage("44", "This is still a PubSubMessage", new Metadata())); List<Tuple> tuples = new ArrayList<>(); expected.forEach(m -> tuples.add(makeTuple(m.getId(), m.getContent(), m.getMetadata()))); for (int i = 0; i < tuples.size(); i++) { bolt.execute(tuples.get(i)); Assert.assertEquals(publisher.getSent().get(i).getId(), expected.get(i).getId()); Assert.assertEquals(publisher.getSent().get(i).getContent(), expected.get(i).getContent()); Assert.assertEquals(publisher.getSent().get(i).getMetadata(), expected.get(i).getMetadata()); Assert.assertTrue(collector.wasNthAcked(tuples.get(i), i + 1)); Assert.assertEquals(collector.getAckedCount(), i + 1); } }
@Test(timeOut = 5000L) public void testException() throws Exception { // This will hit a non-existent url and fail, testing our exceptions. Our connect and retry is low so even if // block the full amount, it's still fast. pubscriber.send(new PubSubMessage("foo", "bar")); PubSubMessage actual = fetchAsync().get(); Assert.assertNotNull(actual); Assert.assertEquals(actual.getId(), "foo"); Assert.assertEquals(actual.getContent(), CANNOT_REACH_DRPC.asJSONClip()); } }
@Test(timeOut = 5000L) public void testReadingOkResponse() throws Exception { PubSubMessage expected = new PubSubMessage("foo", "response"); CompletableFuture<Response> response = getOkFuture(getOkResponse(expected.asJSON())); AsyncHttpClient mockClient = mockClientWith(response); pubscriber.setClient(mockClient); pubscriber.send(new PubSubMessage("foo", "bar")); // This is async (but practically still very fast since we mocked the response), so need a timeout. PubSubMessage actual = fetchAsync().get(); Assert.assertNotNull(actual); Assert.assertEquals(actual.getId(), expected.getId()); Assert.assertEquals(actual.getContent(), expected.getContent()); }
@Test public void testMessagesAreLooped() { List<PubSubMessage> expected = asList(new PubSubMessage("42", Metadata.Signal.KILL), new PubSubMessage("43", Metadata.Signal.COMPLETE), new PubSubMessage("44", (Metadata.Signal) null)); List<Tuple> tuples = new ArrayList<>(); expected.forEach(m -> tuples.add(makeTuple(m.getId(), m.getMetadata()))); for (int i = 0; i < tuples.size(); i++) { bolt.execute(tuples.get(i)); Assert.assertEquals(publisher.getSent().get(i).getId(), expected.get(i).getId()); Assert.assertEquals(publisher.getSent().get(i).getMetadata(), expected.get(i).getMetadata()); Assert.assertTrue(collector.wasNthAcked(tuples.get(i), i + 1)); Assert.assertEquals(collector.getAckedCount(), i + 1); } }
@Override public void send(PubSubMessage message) throws PubSubException { String url = urls.get(); String id = message.getId(); String json = message.asJSON(); log.info("Posting to {} for id {}", url, id); log.debug("Posting to {} with body {}", url, json); client.preparePost(url).setBody(json).execute().toCompletableFuture() .exceptionally(this::handleException) .thenAcceptAsync(createResponseConsumer(id)); }
/** * Send a message with an ID and content. * * @param id The ID associated with the message. * @param content The content of the message. * @throws PubSubException if the messaging system throws an error. */ default void send(String id, String content) throws PubSubException { send(new PubSubMessage(id, content)); }
@Override public void send(PubSubMessage message) { // Put responseURL in the metadata so the ResponsePublisher knows to which host to send the response Metadata metadata = message.getMetadata(); metadata = metadata == null ? new Metadata() : metadata; metadata.setContent(resultURL); message.setMetadata(metadata); sendToURL(queryURL, message); } }
@Override public String toString() { return asJSON(); }
@Override public void send(PubSubMessage message) { String url = (String) message.getMetadata().getContent(); log.debug("Extracted url to which to send results: {}", url); sendToURL(url, message); } }
/** * Check if message has a given {@link Signal}. * * @param signal The signal to check for. * @return true if message has the given signal. */ public boolean hasSignal(Signal signal) { return hasMetadata() && metadata.hasSignal(signal); }
@Override public List<PubSubMessage> getMessages() { List<PubSubMessage> messages = new ArrayList<>(); long currentTime = System.currentTimeMillis(); if (currentTime - lastRequest <= minWait) { return messages; } lastRequest = currentTime; for (String url : urls) { try (CloseableHttpResponse response = client.execute(makeHttpGet(url))) { int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == RESTPubSub.OK_200) { HttpEntity httpEntity = response.getEntity(); String message = EntityUtils.toString(httpEntity, RESTPubSub.UTF_8); log.debug("Received message from url: {}. Message was {}", url, message); messages.add(PubSubMessage.fromJSON(message)); EntityUtils.consume(httpEntity); } else if (statusCode != RESTPubSub.NO_CONTENT_204) { // NO_CONTENT_204 indicates there are no new messages - anything else indicates a problem log.error("HTTP call to {} failed with status code {} and response {}.", url, statusCode, response); } } catch (Exception e) { log.error("HTTP call to {} failed with error:", url, e); } } return messages; }
@Override public void send(PubSubMessage message) throws PubSubException { Metadata metadata = message.getMetadata(); // Remove the content String content = metadata.getContent().toString(); log.debug("Removing metadata {} for result {}@{}: {}", content, message.getId(), message.getSequence(), message.getContent()); metadata.setContent(null); String serializedMessage = message.asJSON(); Tuple tuple = new DRPCTuple(new Values(serializedMessage, content)); // This sends the message through DRPC and not to the collector but it acks or fails accordingly. bolt.execute(tuple); if (!collector.isAcked()) { throw new PubSubException("Message not acked. Unable to send message through DRPC:\n " + serializedMessage); } // Otherwise, we're good to proceed collector.reset(); }
@Test public void testNextTupleMessagesAreReceivedAndTupleIsEmitted() { // Add messages to be received from subscriber PubSubMessage messageA = new PubSubMessage("42", "This is a PubSubMessage", new Metadata()); PubSubMessage messageB = new PubSubMessage("43", "This is also a PubSubMessage", new Metadata()); subscriber.addMessages(messageA, messageB); Assert.assertEquals(subscriber.getReceived().size(), 0); Assert.assertEquals(emitter.getEmitted().size(), 0); // subscriber.receive() -> messageA spout.nextTuple(); Assert.assertEquals(subscriber.getReceived().size(), 1); Assert.assertEquals(subscriber.getReceived().get(0), messageA); Tuple emittedFirst = TupleUtils.makeTuple(TupleClassifier.Type.QUERY_TUPLE, messageA.getId(), messageA.getContent(), messageA.getMetadata()); Assert.assertEquals(emitter.getEmitted().size(), 1); Assert.assertTrue(emitter.wasNthEmitted(emittedFirst, 1)); // subscriber.receive() -> messageB spout.nextTuple(); Assert.assertEquals(subscriber.getReceived().size(), 2); Assert.assertEquals(subscriber.getReceived().get(0), messageA); Assert.assertEquals(subscriber.getReceived().get(1), messageB); Tuple emittedSecond = TupleUtils.makeTuple(TupleClassifier.Type.QUERY_TUPLE, messageB.getId(), messageB.getContent(), messageB.getMetadata()); Assert.assertEquals(emitter.getEmitted().size(), 2); Assert.assertTrue(emitter.wasNthEmitted(emittedFirst, 1)); Assert.assertTrue(emitter.wasNthEmitted(emittedSecond, 2)); }
@Test(timeOut = 5000L) public void testReadingNullResponse() throws Exception { CompletableFuture<Response> response = getOkFuture(null); AsyncHttpClient mockClient = mockClientWith(response); pubscriber.setClient(mockClient); pubscriber.send(new PubSubMessage("foo", "bar")); // This is async (but practically still very fast since we mocked the response), so need a timeout. PubSubMessage actual = fetchAsync().get(); Assert.assertNotNull(actual); Assert.assertEquals(actual.getId(), "foo"); Assert.assertEquals(actual.getContent(), CANNOT_REACH_DRPC.asJSONClip()); }