try { if (consumer == null) { consumer = new SimpleConsumer(brokerHost.getHost(), brokerHost.getPort(), config.socketTimeoutMs, config.socketReceiveBufferBytes, config.clientId); consumer = new SimpleConsumer(host.getHost(), host.getPort(), config.socketTimeoutMs, config.socketReceiveBufferBytes, config.clientId); kafka.javaapi.TopicMetadataResponse resp = null; try { resp = consumer.send(req); } catch (Exception e) { errors += 1; consumer.close(); consumer = null; } finally { if (consumer != null) { consumer.close(); consumer = null;
private static long[] findAllOffsets(SimpleConsumer consumer, String topicName, int partitionId) { TopicAndPartition topicAndPartition = new TopicAndPartition(topicName, partitionId); // The API implies that this will always return all of the offsets. So it seems a partition can not have // more than Integer.MAX_VALUE-1 segments. // // This also assumes that the lowest value returned will be the first segment available. So if segments have been dropped off, this value // should not be 0. PartitionOffsetRequestInfo partitionOffsetRequestInfo = new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), Integer.MAX_VALUE); OffsetRequest offsetRequest = new OffsetRequest(ImmutableMap.of(topicAndPartition, partitionOffsetRequestInfo), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest); if (offsetResponse.hasError()) { short errorCode = offsetResponse.errorCode(topicName, partitionId); throw new RuntimeException("could not fetch data from Kafka, error code is '" + errorCode + "'"); } return offsetResponse.offsets(topicName, partitionId); }
private synchronized FetchResponse getFetchResponseForFetchRequest(FetchRequest fetchRequest, KafkaPartition partition) { SimpleConsumer consumer = getSimpleConsumer(partition.getLeader().getHostAndPort()); FetchResponse fetchResponse = consumer.fetch(fetchRequest); if (fetchResponse.hasError()) { throw new RuntimeException( String.format("error code %d", fetchResponse.errorCode(partition.getTopicName(), partition.getId()))); } return fetchResponse; }
public long getOffset(String topic, int partition, long startOffsetTime) { SimpleConsumer simpleConsumer = findLeaderConsumer(partition); if (simpleConsumer == null) { LOG.error("Error consumer is null get offset from partition:" + partition); return -1; } TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1)); OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), simpleConsumer.clientId()); long[] offsets = simpleConsumer.getOffsetsBefore(request).offsets(topic, partition); if (offsets.length > 0) { return offsets[0]; } else { return NO_OFFSET; } }
@Override public void close() throws IOException { int numOfConsumersNotClosed = 0; for (SimpleConsumer consumer : this.activeConsumers.values()) { if (consumer != null) { try { consumer.close(); } catch (Exception e) { log.warn(String.format("Failed to close Kafka Consumer %s:%d", consumer.host(), consumer.port())); numOfConsumersNotClosed++; } } } this.activeConsumers.clear(); if (numOfConsumersNotClosed > 0) { throw new IOException(numOfConsumersNotClosed + " consumer(s) failed to close."); } }
private static long getLatestOffset(SimpleConsumer consumer, TopicAndPartition topicAndPartition) { Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1)); kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); OffsetResponse response = consumer.getOffsetsBefore(request); if (response.hasError()) { logger.warn("Failed to fetch offset for {} due to {}", topicAndPartition, response.errorCode(topicAndPartition.topic(), topicAndPartition.partition())); return -1; } long[] offsets = response.offsets(topicAndPartition.topic(), topicAndPartition.partition()); return offsets[0]; } }
OffsetResponse response; while (true) { kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest( partitionToRequestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); response = consumer.getOffsetsBefore(request); if (response.hasError()) { StringBuilder exception = new StringBuilder(); for (KafkaTopicPartitionState<TopicAndPartition> part : partitionStates) { short code; if ((code = response.errorCode(part.getTopic(), part.getPartition())) != ErrorMapping.NoError()) { exception.append("\nException for topic=").append(part.getTopic()) .append(" partition=").append(part.getPartition()).append(": ") final long offset = response.offsets(part.getTopic(), part.getPartition())[0];
private long getEarliestOffset() { // return kafka.api.OffsetRequest.EarliestTime(); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfoMap = new HashMap<>(); requestInfoMap.put(topicAndPartition, new PartitionOffsetRequestInfo(EARLIEST_TIME, 1)); OffsetRequest offsetRequest = new OffsetRequest(requestInfoMap, kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID); if (earliestOffset <= 0) { OffsetResponse offsetResponse = consumer.getOffsetsBefore(offsetRequest); earliestOffset = offsetResponse.offsets(topicAndPartition.topic(), topicAndPartition.partition())[0]; } return earliestOffset; }
private SimpleConsumer mockSimpleConsumer(TopicMetadataResponse metadataResponse, OffsetResponse offsetResponse, FetchResponse fetchResponse) { SimpleConsumer simpleConsumer = EasyMock.createMock(SimpleConsumer.class); mocks.add(simpleConsumer); EasyMock.expect(simpleConsumer.send((TopicMetadataRequest)EasyMock.anyObject())).andReturn(metadataResponse).times(1); EasyMock.expect(simpleConsumer.getOffsetsBefore((OffsetRequest)EasyMock.anyObject())).andReturn(offsetResponse).anyTimes(); simpleConsumer.close(); EasyMock.expectLastCall().andVoid().anyTimes(); EasyMock.expect(simpleConsumer.clientId()).andReturn(KAFKA_CLIENT_ID).times(1); EasyMock.expect(simpleConsumer.fetch((FetchRequest)EasyMock.anyObject())).andReturn(fetchResponse).times(1); EasyMock.expect(simpleConsumer.host()).andReturn("dummyHost").anyTimes(); EasyMock.expect(simpleConsumer.port()).andReturn(8888).anyTimes(); return simpleConsumer; }
new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1); new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1); Map<TopicAndPartition, PartitionOffsetRequestInfo> latestOffsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); earliestOffsetResponse = getLatestOffsetResponse(consumer, earliestOffsetInfo, context); consumer.close(); if (earliestOffsetResponse == null) { log.warn(generateLogWarnForSkippedTopics(earliestOffsetInfo, consumer)); long latestOffset = latestOffsetResponse.offsets(topicAndPartition.topic(), topicAndPartition.partition())[0]; long earliestOffset = earliestOffsetResponse.offsets(topicAndPartition.topic(), topicAndPartition.partition())[0];
protected OffsetResponse getLatestOffsetResponse(SimpleConsumer consumer, Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo, JobContext context) { for (int i = 0; i < NUM_TRIES_FETCH_FROM_LEADER; i++) { try { OffsetResponse offsetResponse = consumer.getOffsetsBefore(new OffsetRequest(offsetInfo, kafka.api.OffsetRequest.CurrentVersion(), CamusJob .getKafkaClientName(context))); if (offsetResponse.hasError()) { throw new RuntimeException("offsetReponse has error."); } return offsetResponse; } catch (Exception e) { log.warn("Fetching offset from leader " + consumer.host() + ":" + consumer.port() + " has failed " + (i + 1) + " time(s). Reason: " + e.getMessage() + " " + (NUM_TRIES_FETCH_FROM_LEADER - i - 1) + " retries left."); if (i < NUM_TRIES_FETCH_FROM_LEADER - 1) { try { Thread.sleep((long) (Math.random() * (i + 1) * 1000)); } catch (InterruptedException e1) { log.error("Caught interrupted exception between retries of getting latest offsets. " + e1.getMessage()); } } } } return null; }
consumer = new SimpleConsumer(broker.host(), broker.port(), soTimeout, bufferSize, clientId); fetchResponse = consumer.fetch(fetchRequest); consumer.close(); } catch (Throwable t) { LOG.warn("Error while closing consumer connection", t); consumer = new SimpleConsumer(broker.host(), broker.port(), soTimeout, bufferSize, clientId); continue; // retry } else { consumer.close();
@Override public SimpleConsumer buildSimpleConsumer(String host, int port, int soTimeout, int bufferSize, String clientId) { return new SimpleConsumer(host, port, soTimeout, bufferSize, clientId); } }
SimpleConsumer consumer = createBrokerConsumer(context, brokers.get(i)); log.info(String.format("Fetching metadata from broker %s with client id %s for %d topic(s) %s", brokers.get(i), consumer.clientId(), metaRequestTopics.size(), metaRequestTopics)); try { for (int iter = 0; iter < NUM_TRIES_TOPIC_METADATA; iter++) { try { topicMetadataList = consumer.send(new TopicMetadataRequest(metaRequestTopics)).topicsMetadata(); fetchMetaDataSucceeded = true; break; log.warn(String.format( "Fetching topic metadata with client id %s for topics [%s] from broker [%s] failed, iter[%s]", consumer.clientId(), metaRequestTopics, brokers.get(i), iter), e); try { Thread.sleep((long) (Math.random() * (iter + 1) * 1000)); consumer.close(); i++;
private List<TopicMetadata> fetchTopicMetadataFromBroker(String broker, String... selectedTopics) { LOG.info(String.format("Fetching topic metadata from broker %s", broker)); SimpleConsumer consumer = null; try { consumer = getSimpleConsumer(broker); for (int i = 0; i < this.fetchTopicRetries; i++) { try { return consumer.send(new TopicMetadataRequest(Arrays.asList(selectedTopics))).topicsMetadata(); } catch (Exception e) { LOG.warn(String.format("Fetching topic metadata from broker %s has failed %d times.", broker, i + 1), e); try { Thread.sleep((long) ((i + Math.random()) * 1000)); } catch (InterruptedException e2) { LOG.warn("Caught InterruptedException: " + e2); } } } } finally { if (consumer != null) { consumer.close(); } } return null; }
@Override protected void closeConnections() throws Exception { if (consumer != null) { SimpleConsumer consumer = this.consumer; consumer.close(); // de-reference the consumer to avoid closing multiple times this.consumer = null; } }
TopicAndPartition topicAndPartition = new TopicAndPartition(kafkaRequest.getTopic(), kafkaRequest.getPartition()); log.debug("\nAsking for offset : " + (currentOffset)); PartitionFetchInfo partitionFetchInfo = new PartitionFetchInfo(currentOffset, fetchBufferSize); fetchResponse = simpleConsumer.fetch(fetchRequest); if (fetchResponse.hasError()) { String message =
private void updateLeaderMap() { for (String broker : brokerList) { try { SimpleConsumer consumer = getSimpleConsumer(broker); TopicMetadataRequest req = new TopicMetadataRequest(auditTopics); kafka.javaapi.TopicMetadataResponse resp = consumer.send(req); List<TopicMetadata> metaData = resp.topicsMetadata(); for (TopicMetadata tmd : metaData) { for (PartitionMetadata pmd : tmd.partitionsMetadata()) { TopicAndPartition topicAndPartition = new TopicAndPartition(tmd.topic(), pmd.partitionId()); partitionLeader.put(topicAndPartition, getHostPort(pmd.leader())); } } } catch (Exception e) { logger.warn("Got exception to get metadata from broker=" + broker, e); } } }