void enqueue(ConsumerRecords<String, byte[]> records) { records.forEach((record) -> { TopicPartition partitionKey = new TopicPartition(record.topic(), record.partition()); PartitionProcessor processor = processors.get(partitionKey); if (processor == null) { processor = assignNewPartition(partitionKey); } processor.enqueue(record); }); }
@Test public void exceptionInHandlerShouldConsumeMessage() throws InterruptedException { PartitionProcessor processor = givenAPartionProcessor(); processor.enqueue(testRecordWithOffset(42)); getTestHandler(processor).exceptionToBeThrown = new RuntimeException("BOOM"); getTestHandler(processor).onMessageCalled.await(); shortSleep(); assertTrue(processor.hasUncommittedMessages()); assertEquals(43, processor.getCommitOffsetAndClear()); processor.waitForHandlersToTerminate(1); }
@Test public void consumeOneMessage() throws InterruptedException { PartitionProcessor processor = givenAPartionProcessor(); ConsumerRecord<String, byte[]> record = testRecordWithOffset(999); processor.enqueue(record); (getTestHandler(processor)).onMessageCalled.await(); // message not yet marked as consumed assertEquals(0, processor.numberOfUnprocessedMessages()); assertFalse(processor.hasUncommittedMessages()); assertEquals(-1, processor.getCommitOffsetAndClear()); (getTestHandler(processor)).blockReturnFromOnMessage.countDown(); Thread.sleep(1000); // allow handler to continue assertTrue(processor.hasUncommittedMessages()); assertEquals(1000, processor.getCommitOffsetAndClear()); // getCommitOffsetAndClear clears hasUncommittedMessages assertFalse(processor.hasUncommittedMessages()); assertEquals(1000, processor.getCommitOffsetAndClear()); processor.waitForHandlersToTerminate(50); }
@Test public void afterProcesserIsStoppedMessagesToBeEnqueuedWillBeIgnored() throws InterruptedException { PartitionProcessor processor = givenAPartionProcessor(); processor.stopProcessing(); ConsumerRecord<String, byte[]> record = testRecordWithOffset(999); processor.enqueue(record); // should log a message that messages are ignored (and never committed to Kafka) // assert handler not called assertFalse((getTestHandler(processor)).onMessageCalled.await(100, TimeUnit.MILLISECONDS)); processor.waitForHandlersToTerminate(1); }
@Test public void shutDownEvenIfHandlerIsStillActive() throws InterruptedException { PartitionProcessor processor = givenAPartionProcessor(); ConsumerRecord<String, byte[]> record = testRecordWithOffset(999); processor.enqueue(record); (getTestHandler(processor)).onMessageCalled.await(); processor.stopProcessing(); assertFalse(processor.isTerminated()); processor.waitForHandlersToTerminate(10); // should log a warn about still running handlers assertTrue(processor.isTerminated()); assertFalse(processor.hasUncommittedMessages()); // If handler finishes, it will mark message as committable, but at this time, the processor is probably already removed. (getTestHandler(processor)).blockReturnFromOnMessage.countDown(); shortSleep(); // allow handler to continue assertTrue(processor.hasUncommittedMessages()); }
@Test public void retryFailedMessages() throws InterruptedException { TopicPartition topicKey = new TopicPartition(TOPIC, PARTITION); TypeDictionary typeDictionary = new TestTypeDictionary(); class RetryableTestException extends RuntimeException { } class RetryTestFailureHandler extends DelayAndRetryOnRecoverableErrors { public RetryTestFailureHandler(FailedMessageProcessor fallbackStrategy, RetryDelayer retryStrategy) { super(fallbackStrategy, retryStrategy); } @Override protected boolean isRecoverable(Throwable failureCause) { if (failureCause instanceof RetryableTestException) { return true; } return super.isRecoverable(failureCause); } } FailedMessageProcessor failedMessageProcessor = new RetryTestFailureHandler(new DiscardFailedMessages(), new SimpleRetryDelayer(10, 100)); PartitionProcessor processor = new PartitionProcessor(topicKey, typeDictionary, failedMessageProcessor, null, null); getTestHandler(processor).exceptionToBeThrown = new RetryableTestException(); processor.enqueue(testRecordWithOffset(42)); getTestHandler(processor).blockReturnFromOnMessage.countDown(); getTestHandler(processor).handlerReturnedNormally.await(); processor.waitForHandlersToTerminate(100); assertEquals(2, getTestHandler(processor).nbHandlerInvokations.get()); assertTrue(processor.hasUncommittedMessages()); assertEquals(43, processor.getCommitOffsetAndClear()); }
@Test public void pauseProcessorIfBacklogOfUnconsumedMessagesToBig() throws InterruptedException { PartitionProcessor processor = givenAPartionProcessor(); for (int i = 1; i <= PartitionProcessor.MAX_MESSAGES_IN_FLIGHT + 2; i++) { ConsumerRecord<String, byte[]> record = testRecordWithOffset(i); processor.enqueue(record); } while (processor.numberOfUnprocessedMessages() < PartitionProcessor.MAX_MESSAGES_IN_FLIGHT + 1) { shortSleep(); // allow to fill queue } assertEquals(PartitionProcessor.MAX_MESSAGES_IN_FLIGHT + 1, processor.numberOfUnprocessedMessages()); // 1 message currently in handler assertTrue(processor.isPaused()); // should throttle assertFalse(processor.shouldResume()); (getTestHandler(processor)).onMessageCalled.await(); (getTestHandler(processor)).blockReturnFromOnMessage.countDown(); shortSleep(); // allow handler to continue assertTrue(processor.hasUncommittedMessages()); assertEquals(PartitionProcessor.MAX_MESSAGES_IN_FLIGHT + 3, processor.getCommitOffsetAndClear()); assertFalse(processor.isPaused()); // after emptying the queue, processor should resume assertTrue(processor.shouldResume()); processor.waitForHandlersToTerminate(50); }
getTestHandler(processor).resetExceptionToNull = false; // Never terminate normally processor.enqueue(testRecordWithOffset(42));
@Test public void requestResponseCylce() throws InterruptedException { PartitionProcessor processor = givenAPartionProcessor(); EmptyMessage payload = EmptyMessage.getDefaultInstance(); OrangeContext context = new OrangeContext(); ConsumerRecord<String, byte[]> aRecord; Message sentRequest = Messages.requestFor(Topic.defaultServiceInbox("com.sixt.service.cruft"), Topic.serviceInbox("com.sixt.service.cruft", "trashcan"), "requestKey", payload, context); aRecord = simulateKafkaInTheLoop(sentRequest, 10); processor.enqueue(aRecord); getTestHandler(processor).onMessageCalled.await(); Message receivedRequest = getTestHandler(processor).lastMessage; OrangeContext receivedContext = getTestHandler(processor).lastContext; Message sentReply = Messages.replyTo(receivedRequest, payload, receivedContext); aRecord = simulateKafkaInTheLoop(sentReply, 20); processor.enqueue(aRecord); getTestHandler(processor).blockReturnFromOnMessage.countDown(); shortSleep(); // for the reply handling Message receivedReply = getTestHandler(processor).lastMessage; assertEquals(2, getTestHandler(processor).handledMessages.size()); assertEquals(sentRequest.getMetadata().getReplyTo(), receivedReply.getMetadata().getTopic()); assertEquals(sentRequest.getMetadata().getMessageId(), receivedReply.getMetadata().getRequestCorrelationId()); assertEquals(context.getCorrelationId(), receivedReply.getMetadata().getCorrelationId()); processor.waitForHandlersToTerminate(1); }