private List<TopicMetadata> fetchTopicMetadataFromBroker(String broker, String... selectedTopics) { log.info(String.format("Fetching topic metadata from broker %s", broker)); SimpleConsumer consumer = null; try { consumer = getSimpleConsumer(broker); for (int i = 0; i < this.fetchTopicRetries; i++) { try { return consumer.send(new TopicMetadataRequest(Arrays.asList(selectedTopics))).topicsMetadata(); } catch (Exception e) { log.warn(String.format("Fetching topic metadata from broker %s has failed %d times.", broker, i + 1), e); try { Thread.sleep((long) ((i + Math.random()) * 1000)); } catch (InterruptedException e2) { log.warn("Caught InterruptedException: " + e2); } } } } finally { if (consumer != null) { consumer.close(); } } return null; }
private List<TopicMetadata> fetchTopicMetadataFromBroker(String broker, String... selectedTopics) { LOG.info(String.format("Fetching topic metadata from broker %s", broker)); SimpleConsumer consumer = null; try { consumer = getSimpleConsumer(broker); for (int i = 0; i < this.fetchTopicRetries; i++) { try { return consumer.send(new TopicMetadataRequest(Arrays.asList(selectedTopics))).topicsMetadata(); } catch (Exception e) { LOG.warn(String.format("Fetching topic metadata from broker %s has failed %d times.", broker, i + 1), e); try { Thread.sleep((long) ((i + Math.random()) * 1000)); } catch (InterruptedException e2) { LOG.warn("Caught InterruptedException: " + e2); } } } } finally { if (consumer != null) { consumer.close(); } } return null; }
@Override public int getNumPartitions(String topic) { SimpleConsumer consumer = null; try { consumer = createConsumer( mConfig.getKafkaSeedBrokerHost(), mConfig.getKafkaSeedBrokerPort(), "partitionLookup"); List<String> topics = new ArrayList<String>(); topics.add(topic); TopicMetadataRequest request = new TopicMetadataRequest(topics); TopicMetadataResponse response = consumer.send(request); if (response.topicsMetadata().size() != 1) { throw new RuntimeException("Expected one metadata for topic " + topic + " found " + response.topicsMetadata().size()); } TopicMetadata topicMetadata = response.topicsMetadata().get(0); return topicMetadata.partitionsMetadata().size(); } finally { if (consumer != null) { consumer.close(); } } }
for (TopicMetadata item : consumer.send(new TopicMetadataRequest(Collections.<String>emptyList())).topicsMetadata()) { if (item.errorCode() != ErrorMapping.NoError()) {
try { _leader = null; List<PartitionMetadata> pMetaList = response.topicsMetadata().get(0).partitionsMetadata(); for (PartitionMetadata pMeta : pMetaList) { if (pMeta.partitionId() == _partition) {
List<TopicMetadata> metaData = resp.topicsMetadata(); for (TopicMetadata item : metaData) { for (PartitionMetadata part : item.partitionsMetadata()) {
for (TopicMetadata item : consumer.send(new TopicMetadataRequest(topics)).topicsMetadata()) { if (item.errorCode() != ErrorMapping.NoError()) {
final TopicMetadata topicMetadata = topicMetadataResponse.topicsMetadata().get(0); final short errorCode = topicMetadata.errorCode();
private HostAndPort findLeader(TopicPartition topicPartition) { SimpleConsumer consumer = null; try { LOG.debug("looking up leader for topic {} partition {}", topicPartition.getTopic(), topicPartition.getPartition()); consumer = createConsumer( mConfig.getKafkaSeedBrokerHost(), mConfig.getKafkaSeedBrokerPort(), "leaderLookup"); List<String> topics = new ArrayList<String>(); topics.add(topicPartition.getTopic()); TopicMetadataRequest request = new TopicMetadataRequest(topics); TopicMetadataResponse response = consumer.send(request); List<TopicMetadata> metaData = response.topicsMetadata(); for (TopicMetadata item : metaData) { for (PartitionMetadata part : item.partitionsMetadata()) { if (part.partitionId() == topicPartition.getPartition()) { return HostAndPort.fromParts(part.leader().host(), part.leader().port()); } } } } finally { if (consumer != null) { consumer.close(); } } return null; }
TopicMetadataResponse resp = consumer.send(new TopicMetadataRequest(Collections.singletonList(topic))); List<TopicMetadata> metaData = resp.topicsMetadata(); for (TopicMetadata item : metaData) { if (topic.equals(item.topic())) {
for (TopicMetadata metadata : topicMetadataResponse.topicsMetadata()) { for (PartitionMetadata part : metadata.partitionsMetadata()) { log.debug("Adding Partition %s/%s", metadata.topic(), part.partitionId());
for (int iter = 0; iter < NUM_TRIES_TOPIC_METADATA; iter++) { try { topicMetadataList = consumer.send(new TopicMetadataRequest(metaRequestTopics)).topicsMetadata(); fetchMetaDataSucceeded = true; break;
private Map<String, Long> getTopicOffsets(List<String> topics) { ArrayList<HostAndPort> nodes = new ArrayList<>(config.getNodes()); Collections.shuffle(nodes); SimpleConsumer simpleConsumer = consumerManager.getConsumer(nodes.get(0)); TopicMetadataRequest topicMetadataRequest = new TopicMetadataRequest(topics); TopicMetadataResponse topicMetadataResponse = simpleConsumer.send(topicMetadataRequest); ImmutableMap.Builder<String, Long> builder = ImmutableMap.builder(); for (TopicMetadata metadata : topicMetadataResponse.topicsMetadata()) { for (PartitionMetadata part : metadata.partitionsMetadata()) { LOGGER.debug(format("Adding Partition %s/%s", metadata.topic(), part.partitionId())); Broker leader = part.leader(); if (leader == null) { // Leader election going on... LOGGER.warn(format("No leader for partition %s/%s found!", metadata.topic(), part.partitionId())); } else { HostAndPort leaderHost = HostAndPort.fromParts(leader.host(), leader.port()); SimpleConsumer leaderConsumer = consumerManager.getConsumer(leaderHost); long offset = findAllOffsets(leaderConsumer, metadata.topic(), part.partitionId())[0]; builder.put(metadata.topic(), offset); } } } return builder.build(); } }
private TopicMetadataResponse mockTopicMetaDataResponse() { PartitionMetadata pMeta = EasyMock.createMock(PartitionMetadata.class); mocks.add(pMeta); EasyMock.expect(pMeta.errorCode()).andReturn((short)0).anyTimes(); Broker broker = new Broker(0, "localhost", 2121); EasyMock.expect(pMeta.leader()).andReturn(broker).anyTimes(); EasyMock.expect(pMeta.partitionId()).andReturn(PARTITION_1_ID).anyTimes(); List<PartitionMetadata> partitionMetadatas = new ArrayList<PartitionMetadata>(); partitionMetadatas.add(pMeta); TopicMetadata tMeta = EasyMock.createMock(TopicMetadata.class); mocks.add(tMeta); EasyMock.expect(tMeta.topic()).andReturn(TOPIC_1).anyTimes(); EasyMock.expect(tMeta.errorCode()).andReturn((short)0).anyTimes(); EasyMock.expect(tMeta.partitionsMetadata()).andReturn(partitionMetadatas).anyTimes(); List<TopicMetadata> topicMetadatas = new ArrayList<TopicMetadata>(); topicMetadatas.add(tMeta); TopicMetadataResponse metadataResponse = EasyMock.createMock(TopicMetadataResponse.class); mocks.add(metadataResponse); EasyMock.expect(metadataResponse.topicsMetadata()).andReturn(topicMetadatas).anyTimes(); return metadataResponse; }
private void refreshTopicMetadata() { TopicMetadataRequest request = new TopicMetadataRequest(Collections.singletonList(kafkaRequest.getTopic())); TopicMetadataResponse response; try { response = simpleConsumer.send(request); } catch (Exception e) { log.error("Exception caught when refreshing metadata for topic " + request.topics().get(0) + ": " + e.getMessage()); return; } TopicMetadata metadata = response.topicsMetadata().get(0); for (PartitionMetadata partitionMetadata : metadata.partitionsMetadata()) { if (partitionMetadata.partitionId() == kafkaRequest.getPartition()) { simpleConsumer = new SimpleConsumer(partitionMetadata.leader().host(), partitionMetadata.leader().port(), CamusJob.getKafkaTimeoutValue(context), CamusJob.getKafkaBufferSize(context), CamusJob.getKafkaClientName(context)); break; } } }
private void updateLeaderMap() { for (String broker : brokerList) { try { SimpleConsumer consumer = getSimpleConsumer(broker); TopicMetadataRequest req = new TopicMetadataRequest(auditTopics); kafka.javaapi.TopicMetadataResponse resp = consumer.send(req); List<TopicMetadata> metaData = resp.topicsMetadata(); for (TopicMetadata tmd : metaData) { for (PartitionMetadata pmd : tmd.partitionsMetadata()) { TopicAndPartition topicAndPartition = new TopicAndPartition(tmd.topic(), pmd.partitionId()); partitionLeader.put(topicAndPartition, getHostPort(pmd.leader())); } } } catch (Exception e) { logger.warn("Got exception to get metadata from broker=" + broker, e); } } }
TopicMetadataResponse topicMetadataResponse = EasyMock.createMock(TopicMetadataResponse.class); mocks.add(topicMetadataResponse); EasyMock.expect(topicMetadataResponse.topicsMetadata()).andReturn(topicMetadatas);
SimpleConsumer consumer = new SimpleConsumer(broker.host(), broker.port(), 100000, 64 * 1024, "your_group_id"); List<String> topics = new ArrayList<>(); topics.add(topic); TopicMetadataRequest req = new TopicMetadataRequest(topics); TopicMetadataResponse resp = simpleConsumer.send(req); if (resp.topicsMetadata().size() != 1) { throw new RuntimeException("Expected one metadata for topic " + topic + " found " + resp.topicsMetadata().size()); } TopicMetadata topicMetaData = resp.topicsMetadata().get(0);
@Override public Integer getNumPartitionsForTopic(String topic){ TopicMetadataRequest topicRequest = new TopicMetadataRequest(Arrays.asList(topic)); TopicMetadataResponse topicResponse = simpleConsumer.send(topicRequest); for (TopicMetadata topicMetadata : topicResponse.topicsMetadata()) { if (topic.equals(topicMetadata.topic())) { int partitionSize = topicMetadata.partitionsMetadata().size(); logger.debug("Partition size found ({}) for {} topic", partitionSize, topic); return partitionSize; } } logger.warn("Metadata info not found!. TOPIC {}", topic); return null; }
@Override public Integer getNumPartitionsForTopic(String topic){ TopicMetadataRequest topicRequest = new TopicMetadataRequest(Arrays.asList(topic)); TopicMetadataResponse topicResponse = simpleConsumer.send(topicRequest); for (TopicMetadata topicMetadata : topicResponse.topicsMetadata()) { if (topic.equals(topicMetadata.topic())) { int partitionSize = topicMetadata.partitionsMetadata().size(); logger.debug("Partition size found ({}) for {} topic", partitionSize, topic); return partitionSize; } } logger.warn("Metadata info not found!. TOPIC {}", topic); return null; }