@Override public synchronized long position(TopicPartition partition) { ensureNotClosed(); if (!this.subscriptions.isAssigned(partition)) throw new IllegalArgumentException("You can only check the position for partitions assigned to this consumer."); Long offset = this.subscriptions.position(partition); if (offset == null) { updateFetchPosition(partition); offset = this.subscriptions.position(partition); } return offset; }
@Test(expected = IllegalStateException.class) public void cantChangePositionForNonAssignedPartition() { state.position(tp0, 1); }
@Test(expected = IllegalStateException.class) public void invalidPositionUpdate() { state.subscribe(singleton(topic), rebalanceListener); state.assignFromSubscribed(singleton(tp0)); state.position(tp0, 0); }
private void ensureBlockOnRecord(long blockedOffset) { // the fetchedRecords() should always throw exception due to the invalid message at the starting offset. for (int i = 0; i < 2; i++) { try { fetcher.fetchedRecords(); fail("fetchedRecords should have raised KafkaException"); } catch (KafkaException e) { assertEquals(blockedOffset, subscriptions.position(tp0).longValue()); } } }
@Test public void testUpdateFetchPositionNoOpWithPositionSet() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 5L); fetcher.resetOffsetsIfNeeded(); assertFalse(client.hasInFlightRequests()); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
private void seekAndConsumeRecord(ByteBuffer responseBuffer, long toOffset) { // Seek to skip the bad record and fetch again. subscriptions.seek(tp0, toOffset); // Should not throw exception after the seek. fetcher.fetchedRecords(); assertEquals(1, fetcher.sendFetches()); client.prepareResponse(fullFetchResponse(tp0, MemoryRecords.readableRecords(responseBuffer), Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); List<ConsumerRecord<byte[], byte[]>> records = fetcher.fetchedRecords().get(tp0); assertEquals(1, records.size()); assertEquals(toOffset, records.get(0).offset()); assertEquals(toOffset + 1, subscriptions.position(tp0).longValue()); }
@Test public void testNoCoordinatorDiscoveryIfPositionsKnown() { assertTrue(coordinator.coordinatorUnknown()); subscriptions.assignFromUser(singleton(t1p)); subscriptions.seek(t1p, 500L); coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); assertEquals(Collections.emptySet(), subscriptions.missingFetchPositions()); assertTrue(subscriptions.hasAllFetchPositions()); assertEquals(500L, subscriptions.position(t1p).longValue()); assertTrue(coordinator.coordinatorUnknown()); }
@Test public void testUpdateFetchPositionOfPausedPartitionsWithAValidPosition() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 10); subscriptions.pause(tp0); // paused partition already has a valid position fetcher.resetOffsetsIfNeeded(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertFalse(subscriptions.isFetchable(tp0)); // because tp is paused assertTrue(subscriptions.hasValidPosition(tp0)); assertEquals(10, subscriptions.position(tp0).longValue()); }
@Test public void testRefreshOffset() { client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); subscriptions.assignFromUser(singleton(t1p)); client.prepareResponse(offsetFetchResponse(t1p, Errors.NONE, "", 100L)); coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); assertEquals(Collections.emptySet(), subscriptions.missingFetchPositions()); assertTrue(subscriptions.hasAllFetchPositions()); assertEquals(100L, subscriptions.position(t1p).longValue()); }
@Test public void testUpdateFetchPositionResetToEarliestOffset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.EARLIEST); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.EARLIEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void testUpdateFetchPositionResetToDefaultOffset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.EARLIEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void testUpdateFetchPositionResetToLatestOffset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.LATEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 5L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(5, subscriptions.position(tp0).longValue()); }
@Test public void testStaleOutOfRangeError() { // verify that an out of range error which arrives after a seek // does not cause us to reset our position or throw an exception subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); subscriptions.seek(tp0, 1); consumerClient.poll(time.timer(0)); assertEquals(0, fetcher.fetchedRecords().size()); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertEquals(1, subscriptions.position(tp0).longValue()); }
@Test public void testRefreshOffsetLoadInProgress() { client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); subscriptions.assignFromUser(singleton(t1p)); client.prepareResponse(offsetFetchResponse(Errors.COORDINATOR_LOAD_IN_PROGRESS)); client.prepareResponse(offsetFetchResponse(t1p, Errors.NONE, "", 100L)); coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); assertEquals(Collections.emptySet(), subscriptions.missingFetchPositions()); assertTrue(subscriptions.hasAllFetchPositions()); assertEquals(100L, subscriptions.position(t1p).longValue()); }
@Test public void testFetchDisconnected() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0), true); consumerClient.poll(time.timer(0)); assertEquals(0, fetcher.fetchedRecords().size()); // disconnects should have no affect on subscription state assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(0, subscriptions.position(tp0).longValue()); }
@Test public void partitionAssignment() { state.assignFromUser(singleton(tp0)); assertEquals(singleton(tp0), state.assignedPartitions()); assertEquals(1, state.numAssignedPartitions()); assertFalse(state.hasAllFetchPositions()); state.seek(tp0, 1); assertTrue(state.isFetchable(tp0)); assertEquals(1L, state.position(tp0).longValue()); state.assignFromUser(Collections.<TopicPartition>emptySet()); assertTrue(state.assignedPartitions().isEmpty()); assertEquals(0, state.numAssignedPartitions()); assertFalse(state.isAssigned(tp0)); assertFalse(state.isFetchable(tp0)); }
@Test public void testFetchOffsetOutOfRange() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, fetcher.sendFetches()); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); consumerClient.poll(time.timer(0)); assertEquals(0, fetcher.fetchedRecords().size()); assertTrue(subscriptions.isOffsetResetNeeded(tp0)); assertEquals(null, subscriptions.position(tp0)); }
@Test public void testRefreshOffsetWithNoFetchableOffsets() { client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); subscriptions.assignFromUser(singleton(t1p)); client.prepareResponse(offsetFetchResponse(t1p, Errors.NONE, "", -1L)); coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); assertEquals(Collections.singleton(t1p), subscriptions.missingFetchPositions()); assertEquals(Collections.emptySet(), subscriptions.partitionsNeedingReset(time.milliseconds())); assertFalse(subscriptions.hasAllFetchPositions()); assertEquals(null, subscriptions.position(t1p)); }
@Test public void testRefreshOffsetNotCoordinatorForConsumer() { client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); subscriptions.assignFromUser(singleton(t1p)); client.prepareResponse(offsetFetchResponse(Errors.NOT_COORDINATOR)); client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); client.prepareResponse(offsetFetchResponse(t1p, Errors.NONE, "", 100L)); coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); assertEquals(Collections.emptySet(), subscriptions.missingFetchPositions()); assertTrue(subscriptions.hasAllFetchPositions()); assertEquals(100L, subscriptions.position(t1p).longValue()); }
@Test public void testUpdateFetchPositionOfPausedPartitionsRequiringOffsetReset() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.pause(tp0); // paused partition does not have a valid position subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); client.prepareResponse(listOffsetRequestMatcher(ListOffsetRequest.LATEST_TIMESTAMP), listOffsetResponse(Errors.NONE, 1L, 10L)); fetcher.resetOffsetsIfNeeded(); consumerClient.pollNoWakeup(); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertFalse(subscriptions.isFetchable(tp0)); // because tp is paused assertTrue(subscriptions.hasValidPosition(tp0)); assertEquals(10, subscriptions.position(tp0).longValue()); }