private void ensureTopicCreated(ZkUtils zkUtils, Set<String> allTopics, String topic, long retentionMs, int replicationFactor, int partitionCount) { Properties props = new Properties(); props.setProperty(LogConfig.RetentionMsProp(), Long.toString(retentionMs)); props.setProperty(LogConfig.CleanupPolicyProp(), DEFAULT_CLEANUP_POLICY); if (!allTopics.contains(topic)) { AdminUtils.createTopic(zkUtils, topic, partitionCount, replicationFactor, props, RackAwareMode.Safe$.MODULE$); } else { try { AdminUtils.changeTopicConfig(zkUtils, topic, props); MetadataResponse.TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(JavaConversions.asScalaSet(Collections.singleton(topic)), zkUtils, ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)).head(); maybeIncreaseTopicReplicationFactor(zkUtils, topicMetadata, replicationFactor, topic); maybeIncreaseTopicPartitionCount(zkUtils, topic, topicMetadata, partitionCount); } catch (RuntimeException re) { LOG.error("Skip updating topic " + topic + " configuration due to failure:" + re.getMessage() + "."); } } }
.fetchTopicMetadataFromZk(JavaConversions.asScalaSet(Collections.singleton(topicName)), zkClient); if (topicMetadatas != null && topicMetadatas.size() > 0) { return JavaConversions.asJavaSet(topicMetadatas).iterator().next().partitionsMetadata().size();
public static void createTopicHelper(final String topicName, final int partitions) { Properties topicProps = new Properties(); topicProps.put(TopicConfig.MESSAGE_TIMESTAMP_TYPE_CONFIG, "CreateTime"); topicProps.put(TopicConfig.RETENTION_MS_CONFIG, "-1"); ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(embeddedKafkaCluster.getZkServer().getConnectionString()), false); AdminUtils.createTopic(zkUtils, topicName, partitions, 1, topicProps, RackAwareMode.Disabled$.MODULE$); org.apache.kafka.common.requests.MetadataResponse.TopicMetadata fetchTopicMetadataFromZk = AdminUtils.fetchTopicMetadataFromZk(topicName, zkUtils); logger.info("Topic Metadata: " + fetchTopicMetadataFromZk); }
AdminUtils.fetchTopicMetadataFromZk(topic,zkUtils); Assert.assertEquals(metaData.partitionsMetadata().size(), Integer.parseInt(topicPartitionCount));
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.errorCode()); // not the first try. Sleep a bit Thread.sleep(150); } Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionsMetadata(); firstPart = partitionMetadata.head(); } while (firstPart.errorCode() != 0); return firstPart.leader().get().id(); } finally { zkUtils.close(); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
AdminUtils.fetchTopicMetadataFromZk(topic,zkUtils); Assert.assertEquals(metaData.partitionsMetadata().size(), partionCount);
@BeforeClass public static void initKafka() throws Exception { synchronized (TestKafkaSuit.class) { if (initCount.get() == 0) { ZookeeperTestUtil.setZookeeperSaslTestConfigProps(); System.setProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM, ClassLoader.getSystemResource(LOGIN_CONF_RESOURCE_PATHNAME).getFile()); embeddedKafkaCluster = new EmbeddedKafkaCluster(); Properties topicProps = new Properties(); zkClient = new ZkClient(embeddedKafkaCluster.getZkServer().getConnectionString(), SESSION_TIMEOUT, CONN_TIMEOUT, ZKStringSerializer$.MODULE$); ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(embeddedKafkaCluster.getZkServer().getConnectionString()), false); AdminUtils.createTopic(zkUtils, TestQueryConstants.JSON_TOPIC, 1, 1, topicProps, RackAwareMode.Disabled$.MODULE$); org.apache.kafka.common.requests.MetadataResponse.TopicMetadata fetchTopicMetadataFromZk = AdminUtils .fetchTopicMetadataFromZk(TestQueryConstants.JSON_TOPIC, zkUtils); logger.info("Topic Metadata: " + fetchTopicMetadataFromZk); KafkaMessageGenerator generator = new KafkaMessageGenerator(embeddedKafkaCluster.getKafkaBrokerList(), StringSerializer.class); generator.populateJsonMsgIntoKafka(TestQueryConstants.JSON_TOPIC, NUM_JSON_MSG); } initCount.incrementAndGet(); runningSuite = true; } logger.info("Initialized Embedded Zookeeper and Kafka"); }
@SuppressWarnings("unchecked") public int partitionSize(String topic, ZkUtils zkUtils) { MetadataResponse.TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils); return topicMetadata.partitionMetadata().size(); }
public short errorCodeFromTopicMetadata(String topic, ZkUtils zkUtils) { MetadataResponse.TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils); return topicMetadata.error().code(); }
public String createUniqueTopic(String prefix, int partitions, Properties topicConfig) throws InterruptedException { checkReady(); String topic = (prefix + UUID.randomUUID().toString().substring(0, 5)).replaceAll("[^a-zA-Z0-9._-]", "_"); AdminUtils.createTopic(kafkaServer.zkUtils(), topic, partitions, 1, topicConfig, AdminUtils.createTopic$default$6()); waitForPassing(Duration.ofSeconds(5), () -> { assertTrue(AdminUtils.fetchTopicMetadataFromZk(topic, kafkaServer.zkUtils()) .partitionMetadata().stream() .allMatch(pm -> !pm.leader().isEmpty())); }); return topic; }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.errorCode()); // not the first try. Sleep a bit Thread.sleep(150); } Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionsMetadata(); firstPart = partitionMetadata.head(); } while (firstPart.errorCode() != 0); return firstPart.leader().get().id(); } finally { zkUtils.close(); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.errorCode()); // not the first try. Sleep a bit Thread.sleep(150); } Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionsMetadata(); firstPart = partitionMetadata.head(); } while (firstPart.errorCode() != 0); return firstPart.leader().get().id(); } finally { zkUtils.close(); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { MetadataResponse.PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.error().code()); // not the first try. Sleep a bit Thread.sleep(150); } List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata(); firstPart = partitionMetadata.get(0); } while (firstPart.error().code() != 0); return firstPart.leader().id(); } finally { zkUtils.close(); } }