/** * Retrieve the behaviour of "auto.offset.reset" from the config properties. * A partition needs to fallback to "auto.offset.reset" as default offset when * we can't find offsets in ZK to start from in {@link StartupMode#GROUP_OFFSETS} startup mode. * * @param config kafka consumer properties * @return either OffsetRequest.LatestTime() or OffsetRequest.EarliestTime() */ private static long getInvalidOffsetBehavior(Properties config) { final String val = config.getProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "largest"); if (val.equals("largest") || val.equals("latest")) { // largest is kafka 0.8, latest is kafka 0.9 return OffsetRequest.LatestTime(); } else { return OffsetRequest.EarliestTime(); } } }
/** * For a set of partitions, if a partition is set with the special offsets {@link OffsetRequest#EarliestTime()} * or {@link OffsetRequest#LatestTime()}, replace them with actual offsets requested via a Kafka consumer. * * @param consumer The consumer connected to lead broker * @param partitions The list of partitions we need offsets for */ private static void requestAndSetEarliestOrLatestOffsetsFromKafka( SimpleConsumer consumer, List<KafkaTopicPartitionState<TopicAndPartition>> partitions) throws Exception { Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>(); for (KafkaTopicPartitionState<TopicAndPartition> part : partitions) { if (part.getOffset() == OffsetRequest.EarliestTime() || part.getOffset() == OffsetRequest.LatestTime()) { requestInfo.put(part.getKafkaPartitionHandle(), new PartitionOffsetRequestInfo(part.getOffset(), 1)); } } requestAndSetOffsetsFromKafka(consumer, partitions, requestInfo); }
@Override protected long getEarliestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException { Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo = Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()), new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1)); return getOffset(partition, offsetRequestInfo); }
@Override public long getEarliestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException { Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo = Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()), new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1)); return getOffset(partition, offsetRequestInfo); }
emittingOffset = consumer.getOffset(config.topic, partition, kafka.api.OffsetRequest.EarliestTime()); } else { if (jsonOffset == null) {
if (partition.getOffset() == KafkaTopicPartitionStateSentinel.EARLIEST_OFFSET) { partition.setOffset(OffsetRequest.EarliestTime()); } else if (partition.getOffset() == KafkaTopicPartitionStateSentinel.LATEST_OFFSET) {
offsetRequestTime = kafka.api.OffsetRequest.LatestTime(); } else if (offsetCriteria.isSmallest()) { offsetRequestTime = kafka.api.OffsetRequest.EarliestTime(); } else { throw new IllegalArgumentException("Unknown initial offset value " + offsetCriteria.toString());
private long getOffset(boolean earliest) throws InterruptedException { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); requestInfo.put( topicAndPartition, new PartitionOffsetRequestInfo( earliest ? kafka.api.OffsetRequest.EarliestTime() : kafka.api.OffsetRequest.LatestTime(), 1 ) ); OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientId); OffsetResponse response; try { response = consumer.getOffsetsBefore(request); } catch (Exception e) { ensureNotInterrupted(e); log.error(e, "caught exception in getOffsetsBefore [%s] - [%s]", topic, partitionId); return -1; } if (response.hasError()) { log.error( "error fetching data Offset from the Broker [%s]. reason: [%s]", leaderBroker.host(), response.errorCode(topic, partitionId) ); return -1; } long[] offsets = response.offsets(topic, partitionId); return earliest ? offsets[0] : offsets[offsets.length - 1]; }
new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1); Map<TopicAndPartition, PartitionOffsetRequestInfo> latestOffsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
protected long getEarliestOffset(){ Map<TopicPartition, Long> brokerOffsets = KafkaUtils .getBrokerOffsets(connectionProperties, OffsetRequest.EarliestTime(), topicPartition.topic()); Long offset = brokerOffsets.get(topicPartition); if(offset == null){ LOG.debug("Unable to determine earliest offset for {} so returning earliest {}", topicPartition, OffsetRequest.EarliestTime()); return OffsetRequest.EarliestTime(); } LOG.debug("Earliest offset for {} is {}", topicPartition, offset); return offset; }
/** * Fetches the earliest offset in Kafka. */ public long fetchEarliestOffset() { return fetchOffsetBefore(kafka.api.OffsetRequest.EarliestTime()); }
/** * getOrigin min offset by specific partition * @return min offset */ private long getMinOffset() { return getOffset(kafka.api.OffsetRequest.EarliestTime()); }
/** * Fetches the earliest offset in Kafka. */ public long fetchEarliestOffset() { return fetchOffsetBefore(kafka.api.OffsetRequest.EarliestTime()); }
@Override public Preparer addFromBeginning(String topic, int partition) { TopicPartition topicPartition = new TopicPartition(topic, partition); requests.put(topicPartition, kafka.api.OffsetRequest.EarliestTime()); return this; }
public long getBeginningOffset(String topic, int partitionId) { return getOffsets( kafkaUtils.getLeader(topic, partitionId), topic, partitionId, kafka.api.OffsetRequest.EarliestTime()); }
public static long getMinOffset(SimpleConsumer consumer, String topic, int partition) throws Exception { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfoMap = new HashMap<>(); requestInfoMap.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1)); kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfoMap, kafka.api.OffsetRequest.CurrentVersion(), "find offset " + System.currentTimeMillis()); OffsetResponse response = consumer.getOffsetsBefore(request); if(response.hasError()) { throw new Exception("error fetching data offset , reason = " + response.errorCode(topic, partition)); } long[] offsets = response.offsets(topic, partition); return offsets[0]; }
@Override protected long getEarliestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException { Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo = Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()), new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1)); return getOffset(partition, offsetRequestInfo); }
@Override public long getEarliestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException { Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo = Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()), new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1)); return getOffset(partition, offsetRequestInfo); }
@Override public long getEarliestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException { Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo = Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()), new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1)); return getOffset(partition, offsetRequestInfo); }
@Override protected long getEarliestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException { Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo = Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()), new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1)); return getOffset(partition, offsetRequestInfo); }