static Envelope toKafka(Message message) { Envelope.Builder envelope = Envelope.newBuilder(); Metadata meta = message.getMetadata(); envelope.setMessageId(meta.getMessageId()); // Correlation ids are set when building the message if (!Strings.isNullOrEmpty(meta.getCorrelationId())) { envelope.setCorrelationId(meta.getCorrelationId()); } // Message exchange pattern headers if (meta.getReplyTo() != null) { envelope.setReplyTo(meta.getReplyTo().toString()); } if (!Strings.isNullOrEmpty(meta.getRequestCorrelationId())) { envelope.setRequestCorrelationId(meta.getRequestCorrelationId()); } // Payload (mandatory fields!) envelope.setMessageType(meta.getType().toString()); envelope.setInnerMessage(message.getPayload().toByteString()); // Serialize the proto payload to bytes return envelope.build(); } }
public static Message<? extends com.google.protobuf.Message> replyTo(Message originalRequest, com.google.protobuf.Message protoPayloadMessage, OrangeContext context) { boolean wasReceived = false; // By default, return to sender topic using same partitioning scheme. Topic target = originalRequest.getMetadata().getReplyTo(); String partitionKey = originalRequest.getMetadata().getPartitioningKey(); String messageId = UUID.randomUUID().toString(); String correlationId = context.getCorrelationId(); String requestCorrelationId = originalRequest.getMetadata().getMessageId(); Topic replyTo = null; // not required MessageType type = MessageType.of(protoPayloadMessage); Metadata meta = new Metadata(wasReceived, target, partitionKey, -1, -1, messageId, correlationId, requestCorrelationId, replyTo, type); return new Message<>(protoPayloadMessage, meta); }
public void send(Message message) { String destinationTopic = message.getMetadata().getTopic().toString(); String partitioningKey = message.getMetadata().getPartitioningKey(); Envelope envelope = Messages.toKafka(message); logger.debug(message.getMetadata().getLoggingMarker(), "Sending message {} with key {} to topic {}", message.getMetadata().getType().toString(), partitioningKey, destinationTopic); logger.warn(message.getMetadata().getLoggingMarker(), "Producer interrupted while waiting on future.get() of kafka.send(record). It is unknown if the message has been sent.", ex); } catch (ExecutionException ex) { Throwable cause = ex.getCause(); logger.warn(message.getMetadata().getLoggingMarker(), "Error sending message", cause);
@Test public void requestResponseCylce() throws InterruptedException { PartitionProcessor processor = givenAPartionProcessor(); EmptyMessage payload = EmptyMessage.getDefaultInstance(); OrangeContext context = new OrangeContext(); ConsumerRecord<String, byte[]> aRecord; Message sentRequest = Messages.requestFor(Topic.defaultServiceInbox("com.sixt.service.cruft"), Topic.serviceInbox("com.sixt.service.cruft", "trashcan"), "requestKey", payload, context); aRecord = simulateKafkaInTheLoop(sentRequest, 10); processor.enqueue(aRecord); getTestHandler(processor).onMessageCalled.await(); Message receivedRequest = getTestHandler(processor).lastMessage; OrangeContext receivedContext = getTestHandler(processor).lastContext; Message sentReply = Messages.replyTo(receivedRequest, payload, receivedContext); aRecord = simulateKafkaInTheLoop(sentReply, 20); processor.enqueue(aRecord); getTestHandler(processor).blockReturnFromOnMessage.countDown(); shortSleep(); // for the reply handling Message receivedReply = getTestHandler(processor).lastMessage; assertEquals(2, getTestHandler(processor).handledMessages.size()); assertEquals(sentRequest.getMetadata().getReplyTo(), receivedReply.getMetadata().getTopic()); assertEquals(sentRequest.getMetadata().getMessageId(), receivedReply.getMetadata().getRequestCorrelationId()); assertEquals(context.getCorrelationId(), receivedReply.getMetadata().getCorrelationId()); processor.waitForHandlersToTerminate(1); }
private void deliveryEnded(Message message, boolean deliveryFailed) { logger.debug(message.getMetadata().getLoggingMarker(), "Message {} with offset {} in {}-{} marked as consumed.", message.getMetadata().getType(), message.getMetadata().getOffset(), message.getMetadata().getTopic().toString(), message.getMetadata().getPartitionId()); if (span != null) { if (deliveryFailed) { Tags.ERROR.set(span, true); } span.finish(); } if (handlerTimer != null) { // may be null in case of UnknownMessageHandlerException if (deliveryFailed) { handlerTimer.recordFailure(startTime); } else { handlerTimer.recordSuccess(startTime); } } }
private void deliveryFailed(Message message, Exception failure, boolean tryDeliverMessage) { logger.debug(message.getMetadata().getLoggingMarker(), "Received tryDeliverMessage={} from {}.onFailedMessage({})", tryDeliverMessage, failedMessageProcessor.getClass().getTypeName(), failure.toString()); if (metricsBuilderFactory != null) { GoCounter deliveryFailures = metricsBuilderFactory.newMetric("messaging_consumer_delivery_failures") .withTag("messageType", message.getMetadata().getType().toString()) .withTag("topic", message.getMetadata().getTopic().toString()) .buildCounter(); if(tryDeliverMessage) { deliveryFailures.incSuccess(); } else { deliveryFailures.incFailure(); } } }
private ConsumerRecord<String, byte[]> simulateKafkaInTheLoop(Message message, long offset) { Envelope envelope = Messages.toKafka(message); return new ConsumerRecord<String, byte[]>(message.getMetadata().getTopic().toString(), PARTITION, offset, message.getMetadata().getPartitioningKey(), envelope.toByteArray()); }
@SuppressWarnings("unchecked") private void deliverToMessageHandler(Message message) { boolean tryDeliverMessage = true; boolean deliveryFailed = true; OrangeContext context = message.getMetadata().newContextFromMetadata(); try { while (tryDeliverMessage) { try { MessageType messageType = message.getMetadata().getType(); MessageHandler handler = typeDictionary.messageHandlerFor(messageType); if (handler == null) { throw new UnknownMessageHandlerException(messageType); } deliveryStarted(message, handler, context); // Leave the framework here: hand over execution to service-specific handler. handler.onMessage(message, context); deliveryFailed = false; break; } catch (Exception failure) { // Strategy decides: Should we retry to deliver the failed message? tryDeliverMessage = failedMessageProcessor.onFailedMessage(message, failure); deliveryFailed(message, failure, tryDeliverMessage); } } } finally { // consume the message - even if delivery failed markAsConsumed(message.getMetadata().getOffset()); deliveryEnded(message, deliveryFailed); } }
public static Message<? extends com.google.protobuf.Message> requestFor(Topic target, Topic replyTo, String partitionKey, com.google.protobuf.Message protoPayloadMessage, OrangeContext context) { boolean wasReceived = false; String messageId = UUID.randomUUID().toString(); String correlationId = context.getCorrelationId(); MessageType type = MessageType.of(protoPayloadMessage); // Use default inbox for service. if (replyTo == null) { throw new IllegalArgumentException("replyTo required"); } String requestCorrelationId = ""; // not required Metadata meta = new Metadata(wasReceived, target, partitionKey, -1, -1, messageId, correlationId, requestCorrelationId, replyTo, type); return new Message<>(protoPayloadMessage, meta); }
@Override public boolean onFailedMessage(Message failed, Throwable failureCause) { logger.warn(failed.getMetadata().getLoggingMarker(), "Discarded failing message.", failureCause); return false; } }
private void deliveryStarted(Message message, MessageHandler handler, OrangeContext context) { logger.debug(message.getMetadata().getLoggingMarker(), "Calling {}.onMessage({})", handler.getClass().getTypeName(), message.getMetadata().getType()); if (tracer != null) { span = tracer.buildSpan(message.getMetadata().getType().toString()).start(); Tags.SPAN_KIND.set(span, "consumer"); span.setTag("correlation_id", context.getCorrelationId()); context.setTracingContext(span.context()); } if (metricsBuilderFactory != null) { handlerTimer = metricsBuilderFactory.newMetric("messaging_consumer_message_handler") .withTag("messageType", message.getMetadata().getType().toString()) .withTag("topic", message.getMetadata().getTopic().toString()) .buildTimer(); startTime = handlerTimer.start(); } }
public static Message<? extends com.google.protobuf.Message> oneWayMessage(Topic target, String partitionKey, com.google.protobuf.Message protoPayloadMessage, OrangeContext context) { boolean wasReceived = false; String messageId = UUID.randomUUID().toString(); String correlationId = context.getCorrelationId(); Topic replyTo = null; // not required String requestCorrelationId = ""; // not required MessageType type = MessageType.of(protoPayloadMessage); Metadata meta = new Metadata(wasReceived, target, partitionKey, -1, -1, messageId, correlationId, requestCorrelationId, replyTo, type); return new Message<>(protoPayloadMessage, meta); }
static Message<? extends com.google.protobuf.Message> fromKafka(com.google.protobuf.Message protoMessage, Envelope envelope, ConsumerRecord<String, byte[]> record) { boolean wasReceived = true; Topic topic = new Topic(record.topic()); String partitioningKey = record.key(); int partitionId = record.partition(); long offset = record.offset(); String messageId = envelope.getMessageId(); String correlationId = envelope.getCorrelationId(); MessageType type = MessageType.of(protoMessage); String requestCorrelationId = envelope.getRequestCorrelationId(); Topic replyTo = new Topic(envelope.getReplyTo()); Metadata meta = new Metadata(wasReceived, topic, partitioningKey, partitionId, offset, messageId, correlationId, requestCorrelationId, replyTo, type); return new Message<>(protoMessage, meta); }