@Override public synchronized void seek(TopicPartition partition, long offset) { ensureNotClosed(); subscriptions.seek(partition, offset); }
@Override public void seek(TopicPartition partition, OffsetAndMetadata offsetAndMetadata) { ensureNotClosed(); subscriptions.seek(partition, offsetAndMetadata.offset()); }
private void updateFetchPosition(TopicPartition tp) { if (subscriptions.isOffsetResetNeeded(tp)) { resetOffsetPosition(tp); } else if (!committed.containsKey(tp)) { subscriptions.requestOffsetReset(tp); resetOffsetPosition(tp); } else { subscriptions.seek(tp, committed.get(tp).offset()); } }
@Test public void testFetchOnPausedPartition() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); subscriptions.pause(tp0); assertFalse(fetcher.sendFetches() > 0); assertTrue(client.requests().isEmpty()); }
@Test public void testFetchedRecordsAfterSeek() { subscriptionsNoAutoReset.assignFromUser(singleton(tp0)); subscriptionsNoAutoReset.seek(tp0, 0); assertTrue(fetcherNoAutoReset.sendFetches() > 0); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); consumerClient.poll(time.timer(0)); assertFalse(subscriptionsNoAutoReset.isOffsetResetNeeded(tp0)); subscriptionsNoAutoReset.seek(tp0, 2); assertEquals(0, fetcherNoAutoReset.fetchedRecords().size()); }
@Test public void testFetchDuringRebalance() { subscriptions.subscribe(singleton(topicName), listener); subscriptions.assignFromSubscribed(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); // Now the rebalance happens and fetch positions are cleared subscriptions.assignFromSubscribed(singleton(tp0)); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); // The active fetch should be ignored since its position is no longer valid assertTrue(fetcher.fetchedRecords().isEmpty()); }
@Test public void testUpdateFetchPositionNoOpWithPositionSet() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 5L); fetcher.resetOffsetsIfNeeded(); assertFalse(client.hasInFlightRequests()); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void testAutoCommitDynamicAssignment() { final String consumerId = "consumer"; ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, ConsumerConfig.DEFAULT_EXCLUDE_INTERNAL_TOPICS, true, true); subscriptions.subscribe(singleton(topic1), rebalanceListener); joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p)); subscriptions.seek(t1p, 100); prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE); time.sleep(autoCommitIntervalMs); coordinator.poll(time.timer(Long.MAX_VALUE)); assertFalse(client.hasPendingResponses()); }
@Test public void testFetchSkipsBlackedOutNodes() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); client.blackout(node, 500); assertEquals(0, fetcher.sendFetches()); time.sleep(500); assertEquals(1, fetcher.sendFetches()); }
@Test public void testInFlightFetchOnPausedPartition() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); subscriptions.pause(tp0); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertNull(fetcher.fetchedRecords().get(tp0)); }
@Test public void testNoCoordinatorDiscoveryIfPositionsKnown() { assertTrue(coordinator.coordinatorUnknown()); subscriptions.assignFromUser(singleton(t1p)); subscriptions.seek(t1p, 500L); coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); assertEquals(Collections.emptySet(), subscriptions.missingFetchPositions()); assertTrue(subscriptions.hasAllFetchPositions()); assertEquals(500L, subscriptions.position(t1p).longValue()); assertTrue(coordinator.coordinatorUnknown()); }
private void makeFetchRequestWithIncompleteRecord() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); assertFalse(fetcher.hasCompletedFetches()); MemoryRecords partialRecord = MemoryRecords.readableRecords( ByteBuffer.wrap(new byte[]{0, 0, 0, 0, 0, 0, 0, 0})); client.prepareResponse(fullFetchResponse(tp0, partialRecord, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); }
@Test public void testUpdateFetchPositionOfPausedPartitionsWithAValidPosition() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 10); subscriptions.pause(tp0); // paused partition already has a valid position fetcher.resetOffsetsIfNeeded(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertFalse(subscriptions.isFetchable(tp0)); // because tp is paused assertTrue(subscriptions.hasValidPosition(tp0)); assertEquals(10, subscriptions.position(tp0).longValue()); }
@Test public void testFetchNotLeaderForPartition() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NOT_LEADER_FOR_PARTITION, 100L, 0)); consumerClient.poll(time.timer(0)); assertEquals(0, fetcher.fetchedRecords().size()); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); }
@Test public void partitionPause() { state.assignFromUser(singleton(tp0)); state.seek(tp0, 100); assertTrue(state.isFetchable(tp0)); state.pause(tp0); assertFalse(state.isFetchable(tp0)); state.resume(tp0); assertTrue(state.isFetchable(tp0)); }
@Test public void testFetchDisconnected() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0), true); consumerClient.poll(time.timer(0)); assertEquals(0, fetcher.fetchedRecords().size()); // disconnects should have no affect on subscription state assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(0, subscriptions.position(tp0).longValue()); }
@Test public void testFetchUnknownTopicOrPartition() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.UNKNOWN_TOPIC_OR_PARTITION, 100L, 0)); consumerClient.poll(time.timer(0)); assertEquals(0, fetcher.fetchedRecords().size()); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); }
@Test public void testFetchOffsetOutOfRange() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); consumerClient.poll(time.timer(0)); assertEquals(0, fetcher.fetchedRecords().size()); assertTrue(subscriptions.isOffsetResetNeeded(tp0)); assertEquals(null, subscriptions.position(tp0)); }
@Test public void testFetchError() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); assertFalse(fetcher.hasCompletedFetches()); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NOT_LEADER_FOR_PARTITION, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetcher.fetchedRecords(); assertFalse(partitionRecords.containsKey(tp0)); }
@Test public void testAutoCommitManualAssignment() { ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, ConsumerConfig.DEFAULT_EXCLUDE_INTERNAL_TOPICS, true, true); subscriptions.assignFromUser(singleton(t1p)); subscriptions.seek(t1p, 100); client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE); time.sleep(autoCommitIntervalMs); coordinator.poll(time.timer(Long.MAX_VALUE)); assertFalse(client.hasPendingResponses()); }