@Override public void handleCompletedMetadataResponse(RequestHeader requestHeader, long now, MetadataResponse response) { this.metadataFetchInProgress = false; // If any partition has leader with missing listeners, log a few for diagnosing broker configuration // issues. This could be a transient issue if listeners were added dynamically to brokers. List<TopicPartition> missingListenerPartitions = response.topicMetadata().stream().flatMap(topicMetadata -> topicMetadata.partitionMetadata().stream() .filter(partitionMetadata -> partitionMetadata.error() == Errors.LISTENER_NOT_FOUND) .map(partitionMetadata -> new TopicPartition(topicMetadata.topic(), partitionMetadata.partition()))) .collect(Collectors.toList()); if (!missingListenerPartitions.isEmpty()) { int count = missingListenerPartitions.size(); log.warn("{} partitions have leader brokers without a matching listener, including {}", count, missingListenerPartitions.subList(0, Math.min(10, count))); } // check if any topics metadata failed to get updated Map<String, Errors> errors = response.errors(); if (!errors.isEmpty()) log.warn("Error while fetching metadata with correlation id {} : {}", requestHeader.correlationId(), errors); // don't update the cluster if there are no valid nodes...the topic we want may still be in the process of being // created which means we will get errors and no nodes until it exists if (response.brokers().isEmpty()) { log.trace("Ignoring empty metadata response with correlation id {}.", requestHeader.correlationId()); this.metadata.failedUpdate(now, null); } else { this.metadata.update(response, now); } }
/** * Transform a topic and PartitionMetadata into PartitionInfo * @return */ public static PartitionInfo partitionMetaToInfo(String topic, PartitionMetadata partitionMetadata) { return new PartitionInfo( topic, partitionMetadata.partition(), partitionMetadata.leader(), partitionMetadata.replicas().toArray(new Node[0]), partitionMetadata.isr().toArray(new Node[0]), partitionMetadata.offlineReplicas().toArray(new Node[0])); }
p.add(new MetadataResponse.PartitionMetadata(Errors.NONE, 0, nodes.get(0), Optional.of(5), singletonList(nodes.get(0)), singletonList(nodes.get(0)), Collections.emptyList())); p.add(new MetadataResponse.PartitionMetadata(Errors.NONE, 1, nodes.get(0), Optional.of(5), singletonList(nodes.get(0)), singletonList(nodes.get(0)), Collections.emptyList())); p.add(new MetadataResponse.PartitionMetadata(Errors.LEADER_NOT_AVAILABLE, 2, null, Optional.empty(), singletonList(nodes.get(0)), singletonList(nodes.get(0)), Collections.emptyList())); p.add(new MetadataResponse.PartitionMetadata(Errors.NONE, 3, nodes.get(0), Optional.of(5), singletonList(nodes.get(0)), singletonList(nodes.get(0)), Collections.emptyList())); p.add(new MetadataResponse.PartitionMetadata(Errors.NONE, 4, nodes.get(0), Optional.of(5), singletonList(nodes.get(0)), singletonList(nodes.get(0)), Collections.emptyList()));
AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); int numPartitions = partitionMataData.size(); int numReplica = partitionMataData.get(0).replicas().size(); List<Integer> brokerIdList = listBrokers().stream().map(broker -> broker.getId()).collect(toList()); List partitionIdList = partitionMataData.stream().map(p -> String.valueOf(p.partition())).collect(toList()); String assignmentStr = addPartition.getReplicaAssignment(); String toBeSetReplicaAssignmentStr = "";
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
public String createUniqueTopic(String prefix, int partitions, Properties topicConfig) throws InterruptedException { checkReady(); String topic = (prefix + UUID.randomUUID().toString().substring(0, 5)).replaceAll("[^a-zA-Z0-9._-]", "_"); AdminUtils.createTopic(kafkaServer.zkUtils(), topic, partitions, 1, topicConfig, AdminUtils.createTopic$default$6()); waitForPassing(Duration.ofSeconds(5), () -> { assertTrue(AdminUtils.fetchTopicMetadataFromZk(topic, kafkaServer.zkUtils()) .partitionMetadata().stream() .allMatch(pm -> !pm.leader().isEmpty())); }); return topic; }
int rackCursor = 0; for (MetadataResponse.PartitionMetadata pm : topicMetadata.partitionMetadata()) { if (pm.replicas().size() < replicationFactor) { List<Object> newAssignedReplica = new ArrayList<>(); Set<String> currentOccupiedRack = new HashSet<>(); pm.replicas().forEach(node -> { newAssignedReplica.add(node.id()); currentOccupiedRack.add(rackByBroker.get(node.id())); newReplicaAssignment.put(new TopicAndPartition(topic, pm.partition()), JavaConverters.asScalaIteratorConverter(newAssignedReplica.iterator()).asScala().toSeq());
new MetadataResponse.PartitionMetadata(error, partition, leader, Optional.of(100), replicas, isr, offlineReplicas)); metadata.update(metadataResponse, 10L); assertNotNull(metadata.fetch().partition(tp)); new MetadataResponse.PartitionMetadata(error, partition, leader, Optional.of(99), replicas, Collections.emptyList(), offlineReplicas)); metadata.update(metadataResponse, 20L); assertEquals(metadata.fetch().partition(tp).inSyncReplicas().length, 1); new MetadataResponse.PartitionMetadata(error, partition, leader, Optional.of(100), replicas, Collections.emptyList(), offlineReplicas)); metadata.update(metadataResponse, 20L); assertEquals(metadata.fetch().partition(tp).inSyncReplicas().length, 0); new MetadataResponse.PartitionMetadata(error, partition, leader, Optional.of(99), replicas, isr, offlineReplicas)); metadata.update(metadataResponse, 10L); assertNull(metadata.fetch().partition(tp));
new MetadataResponse.PartitionMetadata(error, partition, leader, Optional.of(100), replicas, isr, offlineReplicas)); metadata.update(metadataResponse, 10L); assertNotNull(metadata.fetch().partition(tp)); new MetadataResponse.PartitionMetadata(error, partition, leader, Optional.of(101), replicas, isr, offlineReplicas)); metadata.update(metadataResponse, 30L); assertNotNull(metadata.fetch().partition(tp));
List<Node> offlineNodes = convertToNodes(brokers, offlineReplicas); partitionMetadata.add(new PartitionMetadata(partitionError, partition, leaderNode, leaderEpoch, replicaNodes, isrNodes, offlineNodes));
Consumer<PartitionInfo> partitionInfoConsumer) { TopicPartition tp = new TopicPartition(topic, partitionMetadata.partition()); if (partitionMetadata.leaderEpoch().isPresent()) { int newEpoch = partitionMetadata.leaderEpoch().get();
MetadataResponse.PartitionMetadata partitionMetadata = new MetadataResponse.PartitionMetadata( Errors.NONE, 0, leader, Optional.of(10), singletonList(leader), singletonList(leader), singletonList(leader));
List<MetadataResponse.PartitionMetadata> altPartitions = new ArrayList<>(); for (MetadataResponse.PartitionMetadata p : partitions) { altPartitions.add(new MetadataResponse.PartitionMetadata( p.error(), p.partition(), p.replicas(), p.isr(), p.offlineReplicas()) );
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }