@Override public void seekToEnd(Collection<TopicPartition> partitions) { delegate.seekToEnd(partitions); }
@Override public long getLatestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException { TopicPartition topicPartition = new TopicPartition(partition.getTopicName(), partition.getId()); this.consumer.assign(Collections.singletonList(topicPartition)); this.consumer.seekToEnd(topicPartition); return this.consumer.position(topicPartition); }
} else if (firstPollOffsetStrategy == LATEST && isFirstPollSinceTopologyWasDeployed) { LOG.debug("First poll for topic partition [{}], seeking to partition end", tp); consumer.seekToEnd(Collections.singleton(tp)); } else if (lastBatchMeta != null) { LOG.debug("First poll for topic partition [{}], using last batch metadata", tp); } else if (firstPollOffsetStrategy == UNCOMMITTED_LATEST) { LOG.debug("First poll for topic partition [{}] with no last batch metadata, seeking to partition end", tp); consumer.seekToEnd(Collections.singleton(tp));
consumer.seekToEnd(topicPartitionList); this.endOffset = consumer.position(topicPartition); LOG.info("End Offset set to [{}]", this.endOffset);
public void maybeSeekToEnd(final String groupId, final Consumer<byte[], byte[]> client, final Set<TopicPartition> intermediateTopicPartitions) { if (intermediateTopicPartitions.size() > 0) { System.out.println("Following intermediate topics offsets will be reset to end (for consumer group " + groupId + ")"); for (final TopicPartition topicPartition : intermediateTopicPartitions) { if (allTopics.contains(topicPartition.topic())) { System.out.println("Topic: " + topicPartition.topic()); } } client.seekToEnd(intermediateTopicPartitions); } }
public void maybeSeekToEnd(final String groupId, final Consumer<byte[], byte[]> client, final Set<TopicPartition> intermediateTopicPartitions) { if (intermediateTopicPartitions.size() > 0) { System.out.println("Following intermediate topics offsets will be reset to end (for consumer group " + groupId + ")"); for (final TopicPartition topicPartition : intermediateTopicPartitions) { if (allTopics.contains(topicPartition.topic())) { System.out.println("Topic: " + topicPartition.topic()); } } client.seekToEnd(intermediateTopicPartitions); } }
public void maybeSeekToEnd(final String groupId, final Consumer<byte[], byte[]> client, final Set<TopicPartition> intermediateTopicPartitions) { if (intermediateTopicPartitions.size() > 0) { System.out.println("Following intermediate topics offsets will be reset to end (for consumer group " + groupId + ")"); for (final TopicPartition topicPartition : intermediateTopicPartitions) { if (allTopics.contains(topicPartition.topic())) { System.out.println("Topic: " + topicPartition.topic()); } } client.seekToEnd(intermediateTopicPartitions); } }
assertThat(captor.getValue()) .isEqualTo(new HashSet<>(Arrays.asList(new TopicPartition("foo", 0), new TopicPartition("foo", 4)))); verify(consumer).seekToEnd(captor.capture()); assertThat(captor.getValue()) .isEqualTo(new HashSet<>(Arrays.asList(new TopicPartition("foo", 1), new TopicPartition("foo", 5))));
@Override public void seekToEnd(Collection<TopicPartition> partitions) { kafkaConsumer.seekToEnd(partitions); }
@Override public void seekToEnd(Collection<TopicPartition> partitions) { consumer.seekToEnd(partitions); }
client.seekToBeginning(inputTopicPartitions); } else if (options.has(toLatestOption)) { client.seekToEnd(inputTopicPartitions); } else if (options.has(shiftByOption)) { shiftOffsetsBy(client, inputTopicPartitions, options.valueOf(shiftByOption));
client.seekToBeginning(inputTopicPartitions); } else if (options.has(toLatestOption)) { client.seekToEnd(inputTopicPartitions); } else if (options.has(shiftByOption)) { shiftOffsetsBy(client, inputTopicPartitions, options.valueOf(shiftByOption));
client.seekToBeginning(inputTopicPartitions); } else if (options.has(toLatestOption)) { client.seekToEnd(inputTopicPartitions); } else if (options.has(shiftByOption)) { shiftOffsetsBy(client, inputTopicPartitions, options.valueOf(shiftByOption));
assertThat(next.topic()).isEqualTo(topic); assertThat(next.partition()).isEqualTo(0); verify(consumer).seekToEnd(captor.capture()); next = captor.getValue().iterator().next(); assertThat(next.topic()).isEqualTo(topic);
@Override public void seekToEnd() { this.consumer.seekToEnd(Collections.singletonList(topicPartition)); }
public static Consumer<String, byte[]> createConsumer(Properties props, String subscribeTopic) throws Exception { TopicPartition topicPartition = new TopicPartition(subscribeTopic, 0); List<TopicPartition> topicPartitions = Arrays.asList(topicPartition); Consumer<String, byte[]> consumer = new KafkaConsumer<>(props); // consumer.subscribe(Arrays.asList(subscribeTopics.split(","))); consumer.assign(topicPartitions); // consumer都是在topo启动时创建。当Topo重启,目前的策略是对于kafka中未处理的msg放弃。不再消费。所以seek to end。 consumer.seekToEnd(topicPartitions); return consumer; } }
@Override public void onPartitionsAssigned(Consumer<?, ?> consumer, Collection<TopicPartition> tps) { if (initialAssignment.getAndSet(false)) { if ("earliest".equals(resetTo)) { consumer.seekToBeginning(tps); } else if ("latest".equals(resetTo)) { consumer.seekToEnd(tps); } } } });
@Override public void onPartitionsAssigned(Consumer<?, ?> consumer, Collection<TopicPartition> tps) { if (initialAssignment.getAndSet(false)) { if ("earliest".equals(resetTo)) { consumer.seekToBeginning(tps); } else if ("latest".equals(resetTo)) { consumer.seekToEnd(tps); } } } });
@Override public KafkaReadStream<K, V> seekToEnd(Set<TopicPartition> topicPartitions, Handler<AsyncResult<Void>> completionHandler) { this.context.runOnContext(r -> { current = null; this.submitTask((consumer, future) -> { consumer.seekToEnd(topicPartitions); if (future != null) { future.complete(); } }, completionHandler); }); return this; }
@Override public void seekToEnd(Collection<TopicPartition> partitions) { _kafkaConsumer.seekToEnd(partitions); for (TopicPartition tp : partitions) { _consumerRecordsProcessor.clear(tp); // We set the high watermark to 0 if user is seeking to end. This is needed to prevent the consumer from // retrieving high watermark from the committed offsets. _consumerRecordsProcessor.setPartitionConsumerHighWaterMark(tp, 0L); } }