private static long[] findAllOffsets(SimpleConsumer consumer, String topicName, int partitionId) { TopicAndPartition topicAndPartition = new TopicAndPartition(topicName, partitionId); // The API implies that this will always return all of the offsets. So it seems a partition can not have // more than Integer.MAX_VALUE-1 segments. // // This also assumes that the lowest value returned will be the first segment available. So if segments have been dropped off, this value // should not be 0. PartitionOffsetRequestInfo partitionOffsetRequestInfo = new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), Integer.MAX_VALUE); OffsetRequest offsetRequest = new OffsetRequest(ImmutableMap.of(topicAndPartition, partitionOffsetRequestInfo), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest); if (offsetResponse.hasError()) { short errorCode = offsetResponse.errorCode(topicName, partitionId); throw new RuntimeException("could not fetch data from Kafka, error code is '" + errorCode + "'"); } return offsetResponse.offsets(topicName, partitionId); }
@Override public long getLastOffset(long time) { SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000, 1024 * 1024, "hadoop-etl"); Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(time, 1)); OffsetResponse response = consumer .getOffsetsBefore(new OffsetRequest(offsetInfo, kafka.api.OffsetRequest.CurrentVersion(), "hadoop-etl")); long[] endOffset = response.offsets(topic, partition); consumer.close(); if (endOffset.length == 0) { log.info("The exception is thrown because the latest offset retunred zero for topic : " + topic + " and partition " + partition); } this.latestOffset = endOffset[0]; return endOffset[0]; }
private static long getLatestOffset(SimpleConsumer consumer, TopicAndPartition topicAndPartition) { Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1)); kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); OffsetResponse response = consumer.getOffsetsBefore(request); if (response.hasError()) { logger.warn("Failed to fetch offset for {} due to {}", topicAndPartition, response.errorCode(topicAndPartition.topic(), topicAndPartition.partition())); return -1; } long[] offsets = response.offsets(topicAndPartition.topic(), topicAndPartition.partition()); return offsets[0]; } }
private long getOffsets(Node leader, String topic, int partitionId, long time) { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId); SimpleConsumer consumer = new SimpleConsumer(leader.host(), leader.port(), 10000, 1024, "Kafka-zk-simpleconsumer"); PartitionOffsetRequestInfo partitionOffsetRequestInfo = new PartitionOffsetRequestInfo(time, 10000); OffsetRequest offsetRequest = new OffsetRequest( ImmutableMap.of(topicAndPartition, partitionOffsetRequestInfo), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest); if (offsetResponse.hasError()) { short errorCode = offsetResponse.errorCode(topic, partitionId); log.warn(format("Offset response has error: %d", errorCode)); throw new ApiException( "could not fetch data from Kafka, error code is '" + errorCode + "'Exception Message:" + offsetResponse.toString()); } long[] offsets = offsetResponse.offsets(topic, partitionId); consumer.close(); return offsets[0]; }
public long getOffset(String topic, int partition, long startOffsetTime) { SimpleConsumer simpleConsumer = findLeaderConsumer(partition); if (simpleConsumer == null) { LOG.error("Error consumer is null get offset from partition:" + partition); return -1; } TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1)); OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), simpleConsumer.clientId()); long[] offsets = simpleConsumer.getOffsetsBefore(request).offsets(topic, partition); if (offsets.length > 0) { return offsets[0]; } else { return NO_OFFSET; } }
ArrayList<CamusRequest> finalRequests = new ArrayList<CamusRequest>(); for (LeaderInfo leader : offsetRequestInfo.keySet()) { SimpleConsumer consumer = new SimpleConsumer(leader.getUri() .getHost(), leader.getUri().getPort(), 30000, "hive_kafka_client"); PartitionOffsetRequestInfo partitionLatestOffsetRequestInfo = new PartitionOffsetRequestInfo( kafka.api.OffsetRequest.LatestTime(), 1); PartitionOffsetRequestInfo partitionEarliestOffsetRequestInfo = new PartitionOffsetRequestInfo( kafka.api.OffsetRequest.EarliestTime(), 1); Map<TopicAndPartition, PartitionOffsetRequestInfo> latestOffsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); Map<TopicAndPartition, PartitionOffsetRequestInfo> earliestOffsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
final SimpleConsumer consumer = getSimpleConsumer(broker); try { topicMetadataResponse = consumer.send(topicMetadataRequest); break; } catch (Exception err) { Arrays.toString(topics), endpoint.host()), err); } finally { consumer.close(); requestInfo = brokerRequests.get(leader); requestInfo.put(new TopicAndPartition(metadata.topic(), partition.partitionId()), new PartitionOffsetRequestInfo( time, 1)); TopicPartition topicPartition = new TopicPartition(topicAndPartition.topic(), topicAndPartition.partition()); long[] offsets = offsetResponse.offsets(topicAndPartition.topic(), topicAndPartition.partition()); long offset; if (time == kafka.api.OffsetRequest.EarliestTime()) throw new IllegalStateException("We requested the earliest offsets for topic [" + topicAndPartition.topic() + "] but Kafka returned no values"); kafka.api.OffsetRequest.EarliestTime(), topicAndPartition.topic());
private long getEarliestOffset() { // return kafka.api.OffsetRequest.EarliestTime(); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfoMap = new HashMap<>(); requestInfoMap.put(topicAndPartition, new PartitionOffsetRequestInfo(EARLIEST_TIME, 1)); OffsetRequest offsetRequest = new OffsetRequest(requestInfoMap, kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID); if (earliestOffset <= 0) { OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest); earliestOffset = offsetResponse.offsets(topicAndPartition.topic(), topicAndPartition.partition())[0]; } return earliestOffset; }
@Override protected long getLatestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException { Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo = Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()), new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1)); return getOffset(partition, offsetRequestInfo); }
new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1); new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1); Map<TopicAndPartition, PartitionOffsetRequestInfo> latestOffsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); earliestOffsetResponse = getLatestOffsetResponse(consumer, earliestOffsetInfo, context); consumer.close(); if (earliestOffsetResponse == null) { log.warn(generateLogWarnForSkippedTopics(earliestOffsetInfo, consumer)); long latestOffset = latestOffsetResponse.offsets(topicAndPartition.topic(), topicAndPartition.partition())[0]; long earliestOffset = earliestOffsetResponse.offsets(topicAndPartition.topic(), topicAndPartition.partition())[0];
@Override public long getEarliestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException { Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo = Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()), new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1)); return getOffset(partition, offsetRequestInfo); }
/** * For a set of partitions, if a partition is set with the special offsets {@link OffsetRequest#EarliestTime()} * or {@link OffsetRequest#LatestTime()}, replace them with actual offsets requested via a Kafka consumer. * * @param consumer The consumer connected to lead broker * @param partitions The list of partitions we need offsets for */ private static void requestAndSetEarliestOrLatestOffsetsFromKafka( SimpleConsumer consumer, List<KafkaTopicPartitionState<TopicAndPartition>> partitions) throws Exception { Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>(); for (KafkaTopicPartitionState<TopicAndPartition> part : partitions) { if (part.getOffset() == OffsetRequest.EarliestTime() || part.getOffset() == OffsetRequest.LatestTime()) { requestInfo.put(part.getKafkaPartitionHandle(), new PartitionOffsetRequestInfo(part.getOffset(), 1)); } } requestAndSetOffsetsFromKafka(consumer, partitions, requestInfo); }
private long getTopicLogSize(String topic, int pid) { Option<Object> o = ZkUtils.getLeaderForPartition(zkClient, topic, pid); if (o.isEmpty() || o.get() == null) { log.error("No broker for partition %s - %s", topic, pid); return 0; } Integer leaderId = Int.unbox(o.get()); SimpleConsumer consumer = consumerMap.get(leaderId); if (consumer == null) { consumer = createSimpleConsumer(leaderId); } // createSimpleConsumer may fail. if (consumer == null) { return 0; } consumerMap.put(leaderId, consumer); TopicAndPartition topicAndPartition = new TopicAndPartition(topic, pid); PartitionOffsetRequestInfo requestInfo = new PartitionOffsetRequestInfo(OffsetRequest.LatestTime(), 1); OffsetRequest request = new OffsetRequest( new Map1<TopicAndPartition, PartitionOffsetRequestInfo>(topicAndPartition, requestInfo), 0, Request.OrdinaryConsumerId() ); OffsetResponse response = consumer.getOffsetsBefore(request); PartitionOffsetsResponse offsetsResponse = response.partitionErrorAndOffsets().get(topicAndPartition).get(); return scala.Long.unbox(offsetsResponse.offsets().head()); }
PartitionOffsetRequestInfo requestInfo = new PartitionOffsetRequestInfo(time, 1); .collect(Collectors.toMap(TopicPartitionVO::getId, partition -> Optional.ofNullable( offsetResponse.offsets(topic.getName(), partition.getId())) .map(Arrays::stream) .orElse(LongStream.empty())
/** * Request offsets before a specific time for a set of partitions, via a Kafka consumer. * * @param consumer The consumer connected to lead broker * @param partitions The list of partitions we need offsets for * @param whichTime The type of time we are requesting. -1 and -2 are special constants (See OffsetRequest) */ private static void requestAndSetSpecificTimeOffsetsFromKafka( SimpleConsumer consumer, List<KafkaTopicPartitionState<TopicAndPartition>> partitions, long whichTime) throws IOException { Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>(); for (KafkaTopicPartitionState<TopicAndPartition> part : partitions) { requestInfo.put(part.getKafkaPartitionHandle(), new PartitionOffsetRequestInfo(whichTime, 1)); } requestAndSetOffsetsFromKafka(consumer, partitions, requestInfo); }
@Override public long getLastOffset(long time) { SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000, 1024 * 1024, "hadoop-etl"); Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(time, 1)); OffsetResponse response = consumer .getOffsetsBefore(new OffsetRequest(offsetInfo, kafka.api.OffsetRequest.CurrentVersion(), "hadoop-etl")); long[] endOffset = response.offsets(topic, partition); consumer.close(); if (endOffset.length == 0) { log.info("The exception is thrown because the latest offset retunred zero for topic : " + topic + " and partition " + partition); } this.latestOffset = endOffset[0]; return endOffset[0]; }
private long getLatestOffset(SimpleConsumer consumer, TopicAndPartition topicAndPartition) { Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1)); kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); OffsetResponse response = consumer.getOffsetsBefore(request); if (response.hasError()) { logger.warn("Failed to fetch offset for {} due to {}", topicAndPartition, response.errorCode(topicAndPartition.topic(), topicAndPartition.partition())); return -1; } long[] offsets = response.offsets(topicAndPartition.topic(), topicAndPartition.partition()); return offsets[0]; }
public static long getOffset(SimpleConsumer consumer, String topic, int partition, long startOffsetTime) { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1)); OffsetRequest request = new OffsetRequest( requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); long[] offsets = consumer.getOffsetsBefore(request).offsets(topic, partition); if (offsets.length > 0) { return offsets[0]; } else { return NO_OFFSET; } }
private long getLatestOffset() { // return kafka.api.OffsetRequest.LatestTime(); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfoMap = new HashMap<>(); requestInfoMap.put(topicAndPartition, new PartitionOffsetRequestInfo(LATEST_TIME, 1)); OffsetRequest offsetRequest = new OffsetRequest(requestInfoMap, kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID); if (latestOffset <= 0) { OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest); latestOffset = offsetResponse.offsets(topicAndPartition.topic(), topicAndPartition.partition())[0]; } return latestOffset; }
@Override public long getLatestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException { Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo = Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()), new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1)); return getOffset(partition, offsetRequestInfo); }