Map<TopicPartition, OffsetAndMetadata> offsetsToBeCommitted() { Map<TopicPartition, OffsetAndMetadata> commitOffsets = new HashMap<>(); processors.forEach((key, processor) -> { if (processor.hasUncommittedMessages()) { commitOffsets.put(key, new OffsetAndMetadata(processor.getCommitOffsetAndClear())); } }); return commitOffsets; }
@Test public void consumeOneMessage() throws InterruptedException { PartitionProcessor processor = givenAPartionProcessor(); ConsumerRecord<String, byte[]> record = testRecordWithOffset(999); processor.enqueue(record); (getTestHandler(processor)).onMessageCalled.await(); // message not yet marked as consumed assertEquals(0, processor.numberOfUnprocessedMessages()); assertFalse(processor.hasUncommittedMessages()); assertEquals(-1, processor.getCommitOffsetAndClear()); (getTestHandler(processor)).blockReturnFromOnMessage.countDown(); Thread.sleep(1000); // allow handler to continue assertTrue(processor.hasUncommittedMessages()); assertEquals(1000, processor.getCommitOffsetAndClear()); // getCommitOffsetAndClear clears hasUncommittedMessages assertFalse(processor.hasUncommittedMessages()); assertEquals(1000, processor.getCommitOffsetAndClear()); processor.waitForHandlersToTerminate(50); }
@Test public void exceptionInHandlerShouldConsumeMessage() throws InterruptedException { PartitionProcessor processor = givenAPartionProcessor(); processor.enqueue(testRecordWithOffset(42)); getTestHandler(processor).exceptionToBeThrown = new RuntimeException("BOOM"); getTestHandler(processor).onMessageCalled.await(); shortSleep(); assertTrue(processor.hasUncommittedMessages()); assertEquals(43, processor.getCommitOffsetAndClear()); processor.waitForHandlersToTerminate(1); }
@Test public void basicLifecycle() throws InterruptedException { PartitionProcessor processor = givenAPartionProcessor(); assertFalse(processor.isTerminated()); assertFalse(processor.isPaused()); assertTrue(processor.shouldResume()); assertEquals(0, processor.numberOfUnprocessedMessages()); assertFalse(processor.hasUncommittedMessages()); assertEquals(-1, processor.getCommitOffsetAndClear()); processor.waitForHandlersToTerminate(1); assertTrue(processor.isTerminated()); assertEquals(0, processor.numberOfUnprocessedMessages()); assertFalse(processor.hasUncommittedMessages()); assertEquals(-1, processor.getCommitOffsetAndClear()); }
@Test public void shutDownEvenIfHandlerIsStillActive() throws InterruptedException { PartitionProcessor processor = givenAPartionProcessor(); ConsumerRecord<String, byte[]> record = testRecordWithOffset(999); processor.enqueue(record); (getTestHandler(processor)).onMessageCalled.await(); processor.stopProcessing(); assertFalse(processor.isTerminated()); processor.waitForHandlersToTerminate(10); // should log a warn about still running handlers assertTrue(processor.isTerminated()); assertFalse(processor.hasUncommittedMessages()); // If handler finishes, it will mark message as committable, but at this time, the processor is probably already removed. (getTestHandler(processor)).blockReturnFromOnMessage.countDown(); shortSleep(); // allow handler to continue assertTrue(processor.hasUncommittedMessages()); }
@Test public void pauseProcessorIfBacklogOfUnconsumedMessagesToBig() throws InterruptedException { PartitionProcessor processor = givenAPartionProcessor(); for (int i = 1; i <= PartitionProcessor.MAX_MESSAGES_IN_FLIGHT + 2; i++) { ConsumerRecord<String, byte[]> record = testRecordWithOffset(i); processor.enqueue(record); } while (processor.numberOfUnprocessedMessages() < PartitionProcessor.MAX_MESSAGES_IN_FLIGHT + 1) { shortSleep(); // allow to fill queue } assertEquals(PartitionProcessor.MAX_MESSAGES_IN_FLIGHT + 1, processor.numberOfUnprocessedMessages()); // 1 message currently in handler assertTrue(processor.isPaused()); // should throttle assertFalse(processor.shouldResume()); (getTestHandler(processor)).onMessageCalled.await(); (getTestHandler(processor)).blockReturnFromOnMessage.countDown(); shortSleep(); // allow handler to continue assertTrue(processor.hasUncommittedMessages()); assertEquals(PartitionProcessor.MAX_MESSAGES_IN_FLIGHT + 3, processor.getCommitOffsetAndClear()); assertFalse(processor.isPaused()); // after emptying the queue, processor should resume assertTrue(processor.shouldResume()); processor.waitForHandlersToTerminate(50); }
@Test public void retryFailedMessages() throws InterruptedException { TopicPartition topicKey = new TopicPartition(TOPIC, PARTITION); TypeDictionary typeDictionary = new TestTypeDictionary(); class RetryableTestException extends RuntimeException { } class RetryTestFailureHandler extends DelayAndRetryOnRecoverableErrors { public RetryTestFailureHandler(FailedMessageProcessor fallbackStrategy, RetryDelayer retryStrategy) { super(fallbackStrategy, retryStrategy); } @Override protected boolean isRecoverable(Throwable failureCause) { if (failureCause instanceof RetryableTestException) { return true; } return super.isRecoverable(failureCause); } } FailedMessageProcessor failedMessageProcessor = new RetryTestFailureHandler(new DiscardFailedMessages(), new SimpleRetryDelayer(10, 100)); PartitionProcessor processor = new PartitionProcessor(topicKey, typeDictionary, failedMessageProcessor, null, null); getTestHandler(processor).exceptionToBeThrown = new RetryableTestException(); processor.enqueue(testRecordWithOffset(42)); getTestHandler(processor).blockReturnFromOnMessage.countDown(); getTestHandler(processor).handlerReturnedNormally.await(); processor.waitForHandlersToTerminate(100); assertEquals(2, getTestHandler(processor).nbHandlerInvokations.get()); assertTrue(processor.hasUncommittedMessages()); assertEquals(43, processor.getCommitOffsetAndClear()); }
assertTrue(processor.hasUncommittedMessages()); assertEquals(43, processor.getCommitOffsetAndClear()); processor.waitForHandlersToTerminate(1);