private void poll() { logger.trace("Polling with timeout: {}ms channel-{}", pollTimeout, getName()); try { records = consumer.poll(Duration.ofMillis(pollTimeout)); recordIterator = records.iterator(); logger.debug("{} returned {} records from last poll", getName(), records.count()); } catch (WakeupException e) { logger.trace("Consumer woken up for channel {}.", getName()); } }
public List<QueuedCommand> getRestoreCommands(final Duration duration) { final List<QueuedCommand> restoreCommands = Lists.newArrayList(); commandConsumer.seekToBeginning( Collections.singletonList(commandTopicPartition)); log.debug("Reading prior command records"); ConsumerRecords<CommandId, Command> records = commandConsumer.poll(duration); while (!records.isEmpty()) { log.debug("Received {} records from poll", records.count()); for (final ConsumerRecord<CommandId, Command> record : records) { if (record.value() == null) { continue; } restoreCommands.add( new QueuedCommand( record.key(), record.value(), Optional.empty())); } records = commandConsumer.poll(duration); } return restoreCommands; }
private ConsumerRecords<String, String> pollConsumerRecords(String topic, int maxIter) { ConsumerRecords<String, String> recs = null; for (int i = 0; i < maxIter; i++) { recs = testUtil.getNextMessageFromConsumer(topic); if (recs.count() > 0) break; try { Thread.sleep(1000L); } catch (InterruptedException e) { // } } return recs; }
/** * Poll more records from the Kafka Broker. * * @throws PollTimeoutException if poll returns 0 record and consumer's position < requested endOffset. */ private void pollRecords() { if (LOG.isTraceEnabled()) { stopwatch.reset().start(); } records = consumer.poll(pollTimeoutDurationMs); if (LOG.isTraceEnabled()) { stopwatch.stop(); LOG.trace("Pulled [{}] records in [{}] ms", records.count(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); } // Fail if we can not poll within one lap of pollTimeoutMs. if (records.isEmpty() && consumer.position(topicPartition) < endOffset) { throw new PollTimeoutException(String.format(ERROR_POLL_TIMEOUT_FORMAT, pollTimeoutMs, topicPartition.toString(), startOffset, consumer.position(topicPartition), endOffset)); } consumerRecordIterator = records.iterator(); consumerPosition = consumer.position(topicPartition); }
/** * Executes a poll on the underlying Kafka Consumer and creates any new * flowfiles necessary or appends to existing ones if in demarcation mode. */ void poll() { /** * Implementation note: * Even if ConsumeKafka is not scheduled to poll due to downstream connection back-pressure is engaged, * for longer than session.timeout.ms (defaults to 10 sec), Kafka consumer sends heartbeat from background thread. * If this situation lasts longer than max.poll.interval.ms (defaults to 5 min), Kafka consumer sends * Leave Group request to Group Coordinator. When ConsumeKafka processor is scheduled again, Kafka client checks * if this client instance is still a part of consumer group. If not, it rejoins before polling messages. * This behavior has been fixed via Kafka KIP-62 and available from Kafka client 0.10.1.0. */ try { final ConsumerRecords<byte[], byte[]> records = kafkaConsumer.poll(10); lastPollEmpty = records.count() == 0; processRecords(records); } catch (final ProcessException pe) { throw pe; } catch (final Throwable t) { this.poison(); throw t; } }
@Test public void testConsumerRecordsIsEmptyWhenReturningNoRecords() { TopicPartition partition = new TopicPartition("test", 0); consumer.assign(Collections.singleton(partition)); consumer.addRecord(new ConsumerRecord<String, String>("test", 0, 0, null, null)); consumer.updateEndOffsets(Collections.singletonMap(partition, 1L)); consumer.seekToEnd(Collections.singleton(partition)); ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1)); assertThat(records.count(), is(0)); assertThat(records.isEmpty(), is(true)); }
private ConsumerRecords<K, V> pollKafkaBroker(PollablePartitionsInfo pollablePartitionsInfo) { doSeekRetriableTopicPartitions(pollablePartitionsInfo.pollableEarliestRetriableOffsets); Set<TopicPartition> pausedPartitions = new HashSet<>(consumer.assignment()); pausedPartitions.removeIf(pollablePartitionsInfo.pollablePartitions::contains); try { consumer.pause(pausedPartitions); final ConsumerRecords<K, V> consumerRecords = consumer.poll(kafkaSpoutConfig.getPollTimeoutMs()); ackRetriableOffsetsIfCompactedAway(pollablePartitionsInfo.pollableEarliestRetriableOffsets, consumerRecords); final int numPolledRecords = consumerRecords.count(); LOG.debug("Polled [{}] records from Kafka", numPolledRecords); if (kafkaSpoutConfig.getProcessingGuarantee() == KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE) { //Commit polled records immediately to ensure delivery is at-most-once. Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = createFetchedOffsetsMetadata(consumer.assignment()); consumer.commitSync(offsetsToCommit); LOG.debug("Committed offsets {} to Kafka", offsetsToCommit); } return consumerRecords; } finally { consumer.resume(pausedPartitions); } }
private void checkMessageArrived(String msg, String topic) { ConsumerRecords recs = pollConsumerRecords(topic); assertNotNull(recs); assertTrue(recs.count() > 0); ConsumerRecord consumerRecord = (ConsumerRecord) recs.iterator().next(); assertEquals(msg, consumerRecord.value()); }
@Test public void testSimpleMock() { consumer.subscribe(Collections.singleton("test")); assertEquals(0, consumer.poll(Duration.ZERO).count()); consumer.rebalance(Arrays.asList(new TopicPartition("test", 0), new TopicPartition("test", 1))); // Mock consumers need to seek manually since they cannot automatically reset offsets HashMap<TopicPartition, Long> beginningOffsets = new HashMap<>(); beginningOffsets.put(new TopicPartition("test", 0), 0L); beginningOffsets.put(new TopicPartition("test", 1), 0L); consumer.updateBeginningOffsets(beginningOffsets); consumer.seek(new TopicPartition("test", 0), 0); ConsumerRecord<String, String> rec1 = new ConsumerRecord<>("test", 0, 0, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, "key1", "value1"); ConsumerRecord<String, String> rec2 = new ConsumerRecord<>("test", 0, 1, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, "key2", "value2"); consumer.addRecord(rec1); consumer.addRecord(rec2); ConsumerRecords<String, String> recs = consumer.poll(Duration.ofMillis(1)); Iterator<ConsumerRecord<String, String>> iter = recs.iterator(); assertEquals(rec1, iter.next()); assertEquals(rec2, iter.next()); assertFalse(iter.hasNext()); assertEquals(2L, consumer.position(new TopicPartition("test", 0))); consumer.commitSync(); assertEquals(2L, consumer.committed(new TopicPartition("test", 0)).offset()); }
private void checkData() { Set<TopicPartition> assignment = Collections.singleton(new TopicPartition(TOPIC, 0)); consumer.assign(assignment); consumer.seekToBeginning(assignment); long numRecords = 0; boolean emptyPoll = false; while (numRecords < RECORD_NUMBER && !emptyPoll) { ConsumerRecords<byte[], byte[]> records = consumer.poll(Duration.ofMillis(1000)); Assert.assertFalse(records.records(new TopicPartition(TOPIC, 0)) .stream() .anyMatch(consumerRecord -> !RECORDS_WRITABLES.contains(new KafkaWritable(0, consumerRecord.timestamp(), consumerRecord.value(), consumerRecord.key())))); emptyPoll = records.isEmpty(); numRecords += records.count(); } Assert.assertEquals(RECORD_NUMBER, numRecords); } }
@SuppressWarnings("deprecation") @Test public void testSimpleMockDeprecated() { consumer.subscribe(Collections.singleton("test")); assertEquals(0, consumer.poll(1000).count()); consumer.rebalance(Arrays.asList(new TopicPartition("test", 0), new TopicPartition("test", 1))); // Mock consumers need to seek manually since they cannot automatically reset offsets HashMap<TopicPartition, Long> beginningOffsets = new HashMap<>(); beginningOffsets.put(new TopicPartition("test", 0), 0L); beginningOffsets.put(new TopicPartition("test", 1), 0L); consumer.updateBeginningOffsets(beginningOffsets); consumer.seek(new TopicPartition("test", 0), 0); ConsumerRecord<String, String> rec1 = new ConsumerRecord<>("test", 0, 0, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, "key1", "value1"); ConsumerRecord<String, String> rec2 = new ConsumerRecord<>("test", 0, 1, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, "key2", "value2"); consumer.addRecord(rec1); consumer.addRecord(rec2); ConsumerRecords<String, String> recs = consumer.poll(1); Iterator<ConsumerRecord<String, String>> iter = recs.iterator(); assertEquals(rec1, iter.next()); assertEquals(rec2, iter.next()); assertFalse(iter.hasNext()); assertEquals(2L, consumer.position(new TopicPartition("test", 0))); consumer.commitSync(); assertEquals(2L, consumer.committed(new TopicPartition("test", 0)).offset()); }
@Test public void verifyNoCoordinatorLookupForManualAssignmentWithSeek() { Time time = new MockTime(); Metadata metadata = createMetadata(); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); PartitionAssignor assignor = new RoundRobinAssignor(); KafkaConsumer<String, String> consumer = newConsumer(time, client, metadata, assignor, true); consumer.assign(singleton(tp0)); consumer.seekToBeginning(singleton(tp0)); // there shouldn't be any need to lookup the coordinator or fetch committed offsets. // we just lookup the starting position and send the record fetch. client.prepareResponse(listOffsetsResponse(Collections.singletonMap(tp0, 50L))); client.prepareResponse(fetchResponse(tp0, 50L, 5)); ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1)); assertEquals(5, records.count()); assertEquals(55L, consumer.position(tp0)); consumer.close(Duration.ofMillis(0)); }
@Test public void fetchResponseWithUnexpectedPartitionIsIgnored() { Time time = new MockTime(); Metadata metadata = createMetadata(); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); Node node = metadata.fetch().nodes().get(0); PartitionAssignor assignor = new RangeAssignor(); KafkaConsumer<String, String> consumer = newConsumer(time, client, metadata, assignor, true); consumer.subscribe(singletonList(topic), getConsumerRebalanceListener(consumer)); prepareRebalance(client, node, assignor, singletonList(tp0), null); Map<TopicPartition, FetchInfo> fetches1 = new HashMap<>(); fetches1.put(tp0, new FetchInfo(0, 1)); fetches1.put(t2p0, new FetchInfo(0, 10)); // not assigned and not fetched client.prepareResponseFrom(fetchResponse(fetches1), node); consumer.updateAssignmentMetadataIfNeeded(time.timer(Long.MAX_VALUE)); ConsumerRecords<String, String> records = consumer.poll(Duration.ZERO); assertEquals(0, records.count()); consumer.close(Duration.ofMillis(0)); }
assertEquals(5, records.count()); assertEquals(singleton(tp0), records.partitions());
ConsumerRecords<Integer, Integer> consumerRecords = new ConsumerRecords<>(records); ConsumerRecords<Integer, Integer> interceptedRecords = interceptors.onConsume(consumerRecords); assertEquals(1, interceptedRecords.count()); assertTrue(interceptedRecords.partitions().contains(tp)); assertFalse(interceptedRecords.partitions().contains(filterTopicPart1)); assertEquals(2, partInterceptedRecs.count()); ConsumerRecords<Integer, Integer> noneInterceptedRecs = interceptors.onConsume(consumerRecords); assertEquals(noneInterceptedRecs, consumerRecords); assertEquals(3, noneInterceptedRecs.count()); assertEquals(6, onConsumeCount);
assertEquals(5, records.count());
assertEquals(1, records.count()); assertEquals(11L, consumer.position(tp0));
assertEquals(1, records.count()); assertEquals(11L, consumer.position(tp0));
@Test public void testEmptyChannel() throws EventDeliveryException { Sink kafkaSink = new KafkaSink(); Context context = prepareDefaultContext(); Configurables.configure(kafkaSink, context); Channel memoryChannel = new MemoryChannel(); Configurables.configure(memoryChannel, context); kafkaSink.setChannel(memoryChannel); kafkaSink.start(); Sink.Status status = kafkaSink.process(); if (status != Sink.Status.BACKOFF) { fail("Error Occurred"); } ConsumerRecords recs = pollConsumerRecords(DEFAULT_TOPIC, 2); assertNotNull(recs); assertEquals(recs.count(), 0); }