@Nonnull @Override protected List<OrderedPartitionableRecord<Integer, Long>> getRecords( RecordSupplier<Integer, Long> recordSupplier, TaskToolbox toolbox ) throws Exception { // Handles OffsetOutOfRangeException, which is thrown if the seeked-to // offset is not present in the topic-partition. This can happen if we're asking a task to read from data // that has not been written yet (which is totally legitimate). So let's wait for it to show up. List<OrderedPartitionableRecord<Integer, Long>> records = new ArrayList<>(); try { records = recordSupplier.poll(task.getIOConfig().getPollTimeout()); } catch (OffsetOutOfRangeException e) { log.warn("OffsetOutOfRangeException with message [%s]", e.getMessage()); possiblyResetOffsetsOrWait(e.offsetOutOfRangePartitions(), recordSupplier, toolbox); } return records; }
for (final ConsumerRecord<K, V> rec : recs) { if (beginningOffsets.get(entry.getKey()) != null && beginningOffsets.get(entry.getKey()) > subscriptions.position(entry.getKey())) { throw new OffsetOutOfRangeException(Collections.singletonMap(entry.getKey(), subscriptions.position(entry.getKey())));
assertTrue(e.offsetOutOfRangePartitions().containsKey(tp0)); assertEquals(e.offsetOutOfRangePartitions().size(), 1);
public static ConsumerRecords<byte[], byte[]> fetchMessages( KafkaConfig config, KafkaConsumer<byte[], byte[]> consumer, Partition partition, long offset) { String topic = (String) config._stateConf.get(Config.KAFKA_TOPIC); int partitionId = partition.partition; TopicPartition topicAndPartition = new TopicPartition (topic, partitionId); consumer.seek(topicAndPartition, offset); ConsumerRecords<byte[], byte[]> records; try { records = consumer.poll(config._fillFreqMs / 2); } catch(OffsetOutOfRangeException oore) { throw new OutOfRangeException(oore.getMessage()); } catch (Exception e) { if (e instanceof ConnectException || e instanceof SocketTimeoutException || e instanceof IOException || e instanceof UnresolvedAddressException) { LOG.warn("Network error when fetching messages:", e); throw new FailedFetchException(e); } else { throw new RuntimeException(e); } } return records; } }
assertTrue(oor.offsetOutOfRangePartitions().containsKey(tp0)); assertEquals(oor.offsetOutOfRangePartitions().size(), 1);
log.warn("OffsetOutOfRangeException with message [%s]", e.getMessage()); possiblyResetOffsetsOrWait(e.offsetOutOfRangePartitions(), consumer, toolbox); stillReading = !assignment.isEmpty();
@Test public void testFetchOffsetOutOfRangeException() { subscriptionsNoAutoReset.assignFromUser(singleton(tp0)); subscriptionsNoAutoReset.seek(tp0, 0); fetcherNoAutoReset.sendFetches(); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); consumerClient.poll(time.timer(0)); assertFalse(subscriptionsNoAutoReset.isOffsetResetNeeded(tp0)); for (int i = 0; i < 2; i++) { try { fetcherNoAutoReset.fetchedRecords(); fail("Should have thrown OffsetOutOfRangeException"); } catch (OffsetOutOfRangeException e) { assertTrue(e.offsetOutOfRangePartitions().containsKey(tp0)); assertEquals(e.offsetOutOfRangePartitions().size(), 1); } } }
subscriptions.requestOffsetReset(tp); } else { throw new OffsetOutOfRangeException(Collections.singletonMap(tp, fetchOffset));
log.warn("OffsetOutOfRangeException with message [%s]", e.getMessage()); possiblyResetOffsetsOrWait(e.offsetOutOfRangePartitions(), consumer, toolbox); stillReading = !assignment.isEmpty();
Map<TopicPartition, Long> endOffsets = endOffsets(oe.partitions()); ((OffsetOutOfRangeException) oe).offsetOutOfRangePartitions().forEach((tp, fetchedOffset) -> { long beginningOffset = beginningOffsets.getOrDefault(tp, -1L); long endOffset = endOffsets.getOrDefault(tp, -1L);
log.warn("OffsetOutOfRangeException with message [%s]", e.getMessage()); possiblyResetOffsetsOrWait(e.offsetOutOfRangePartitions(), consumer, toolbox); stillReading = !assignment.isEmpty();
for (final TopicPartition topicPartition : outOfRangeException.offsetOutOfRangePartitions().keySet()) { final long exceptionOffset = outOfRangeException.offsetOutOfRangePartitions().get(topicPartition);