@Override public AdminClient getAdminClient(Map<String, Object> config) { return AdminClient.create(config); }
/** * Create a Kafka topic with the given parameters. * * @param topic The name of the topic. * @param partitions The number of partitions for this topic. * @param replication The replication factor for (partitions of) this topic. * @param topicConfig Additional topic-level configuration settings. */ void createTopic(final String topic, final int partitions, final int replication, final Map<String, String> topicConfig) { log.debug("Creating topic { name: {}, partitions: {}, replication: {}, config: {} }", topic, partitions, replication, topicConfig); final ImmutableMap<String, Object> props = ImmutableMap.of( AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList(), AdminClientConfig.RETRIES_CONFIG, 5); try (AdminClient adminClient = AdminClient.create(props)) { final NewTopic newTopic = new NewTopic(topic, partitions, (short) replication); newTopic.configs(topicConfig); try { final CreateTopicsResult result = adminClient.createTopics(ImmutableList.of(newTopic)); result.all().get(); } catch (final Exception e) { throw new RuntimeException("Failed to create topic:" + topic, e); } } }
/** * Close the AdminClient and release all associated resources. * * See {@link AdminClient#close(long, TimeUnit)} */ @Override public void close() { close(Long.MAX_VALUE, TimeUnit.MILLISECONDS); }
private Config getKafkaBrokerConfig(AdminClient admin) throws Exception { final Collection<Node> nodes = admin.describeCluster().nodes().get(KAFKA_QUERY_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); if (nodes.isEmpty()) { throw new ConnectException("No brokers available to obtain default settings"); } String nodeId = nodes.iterator().next().idString(); Set<ConfigResource> resources = Collections.singleton(new ConfigResource(ConfigResource.Type.BROKER, nodeId)); final Map<ConfigResource, Config> configs = admin.describeConfigs(resources).all().get( KAFKA_QUERY_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS ); if (configs.isEmpty()) { throw new ConnectException("No configs have been received"); } return configs.values().iterator().next(); } }
private void tryDelete(AdminClient adminClient, String topic) throws Exception { try { adminClient.deleteTopics(Collections.singleton(topic)).all().get(DELETE_TIMEOUT_SECONDS, TimeUnit.SECONDS); } catch (TimeoutException e) { LOG.info("Did not receive delete topic response within %d seconds. Checking if it succeeded", DELETE_TIMEOUT_SECONDS); if (adminClient.listTopics().names().get(DELETE_TIMEOUT_SECONDS, TimeUnit.SECONDS).contains(topic)) { throw new Exception("Topic still exists after timeout"); } } }
private void waitForTopics(String bootstrapServers) throws Exception { while (true) { TimeUnit.SECONDS.sleep(5); Properties properties = new Properties(); properties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); AdminClient adminClient = AdminClient.create(properties); if (adminClient.listTopics().names().get().containsAll(TOPICS)) { return; } System.out.println("Waiting for data"); } }
public static void createTopic(String topic, int numPartitions, short replicationFactor, Properties props) { NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor); AdminClient adminClient = AdminClient.create(props); adminClient.createTopics(Collections.singletonList(newTopic)); adminClient.close(); } }
/** * Create a batch of new topics with the default options. * * This is a convenience method for #{@link #createTopics(Collection, CreateTopicsOptions)} with default options. * See the overload for more details. * * This operation is supported by brokers with version 0.10.1.0 or higher. * * @param newTopics The new topics to create. * @return The CreateTopicsResult. */ public CreateTopicsResult createTopics(Collection<NewTopic> newTopics) { return createTopics(newTopics, new CreateTopicsOptions()); }
/** * Create an {@link AdminClient}; invoke the callback and reliably close the admin. * @param callback the callback. */ public void doWithAdmin(java.util.function.Consumer<AdminClient> callback) { Map<String, Object> adminConfigs = new HashMap<>(); adminConfigs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, getBrokersAsString()); AdminClient admin = null; try { admin = AdminClient.create(adminConfigs); callback.accept(admin); } finally { if (admin != null) { admin.close(this.adminTimeout, TimeUnit.SECONDS); } } }
@Test public void testAdminClientWithInvalidCredentials() { Map<String, Object> props = new HashMap<>(saslClientConfigs); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:" + server.port()); try (AdminClient client = AdminClient.create(props)) { DescribeTopicsResult result = client.describeTopics(Collections.singleton("test")); result.all().get(); fail("Expected an authentication error!"); } catch (Exception e) { assertTrue("Expected SaslAuthenticationException, got " + e.getCause().getClass(), e.getCause() instanceof SaslAuthenticationException); } }
@Test public void testAddTopics() throws Exception { AdminClient adminClient = AdminClient.create(this.admin.getConfig()); DescribeTopicsResult topics = adminClient.describeTopics(Arrays.asList("foo", "bar")); topics.all().get(); new DirectFieldAccessor(this.topic1).setPropertyValue("numPartitions", 2); new DirectFieldAccessor(this.topic2).setPropertyValue("numPartitions", 3); this.admin.initialize(); topics = adminClient.describeTopics(Arrays.asList("foo", "bar")); Map<String, TopicDescription> results = topics.all().get(); results.forEach((name, td) -> assertThat(td.partitions()).hasSize(name.equals("foo") ? 2 : 3)); adminClient.close(10, TimeUnit.SECONDS); }
@SuppressWarnings("TryFinallyCanBeTryWithResources") public static boolean kafkaDetected() { AdminClient client = AdminClient.create(getDefaultAdminProperties()); try { client.describeCluster().nodes().get(5, TimeUnit.SECONDS); return true; } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new StreamRuntimeException(e); } catch (ExecutionException e) { throw new StreamRuntimeException(e); } catch (TimeoutException e) { return false; } finally { // cannot use try with resource because of timeout client.close(0, TimeUnit.SECONDS); } }
@Test public void testInvalidTopicNames() throws Exception { try (AdminClientUnitTestEnv env = mockClientEnv()) { env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); List<String> sillyTopicNames = asList("", null); Map<String, KafkaFuture<Void>> deleteFutures = env.adminClient().deleteTopics(sillyTopicNames).values(); for (String sillyTopicName : sillyTopicNames) { TestUtils.assertFutureError(deleteFutures.get(sillyTopicName), InvalidTopicException.class); } assertEquals(0, env.kafkaClient().inFlightRequestCount()); Map<String, KafkaFuture<TopicDescription>> describeFutures = env.adminClient().describeTopics(sillyTopicNames).values(); for (String sillyTopicName : sillyTopicNames) { TestUtils.assertFutureError(describeFutures.get(sillyTopicName), InvalidTopicException.class); } assertEquals(0, env.kafkaClient().inFlightRequestCount()); List<NewTopic> newTopics = new ArrayList<>(); for (String sillyTopicName : sillyTopicNames) { newTopics.add(new NewTopic(sillyTopicName, 1, (short) 1)); } Map<String, KafkaFuture<Void>> createFutures = env.adminClient().createTopics(newTopics).values(); for (String sillyTopicName : sillyTopicNames) { TestUtils.assertFutureError(createFutures .get(sillyTopicName), InvalidTopicException.class); } assertEquals(0, env.kafkaClient().inFlightRequestCount()); } }
public void createTopics(List<String> topicNames, int numPartitions) { List<NewTopic> newTopics = new ArrayList<>(); for (String topicName: topicNames) { NewTopic newTopic = new NewTopic(topicName, numPartitions, (short) 1); newTopics.add(newTopic); } getAdminClient().createTopics(newTopics); //the following lines are a bit of black magic to ensure the topic is ready when we return DescribeTopicsResult dtr = getAdminClient().describeTopics(topicNames); try { dtr.all().get(10, TimeUnit.SECONDS); } catch (Exception e) { throw new RuntimeException("Error getting topic info", e); } } public void deleteTopic(String topicName) {
/** * Describe some topics in the cluster, with the default options. * * This is a convenience method for #{@link AdminClient#describeTopics(Collection, DescribeTopicsOptions)} with * default options. See the overload for more details. * * @param topicNames The names of the topics to describe. * * @return The DescribeTopicsResult. */ public DescribeTopicsResult describeTopics(Collection<String> topicNames) { return describeTopics(topicNames, new DescribeTopicsOptions()); }
@Override public Mono<Health> health() { Health.Builder builder = new Health.Builder(); Properties properties = new Properties(); properties.put("bootstrap.servers", props.getBootstrapServers()); try (AdminClient adminClient = AdminClient.create(properties)) { DescribeClusterResult result = adminClient.describeCluster(new DescribeClusterOptions().timeoutMs(3000)); builder.withDetail("clusterId", result.clusterId().get()); builder.up(); } catch (Exception e) { builder.down(); } return Mono.just(builder.build()); } }
/** * List the topics available in the cluster with the default options. * * This is a convenience method for #{@link AdminClient#listTopics(ListTopicsOptions)} with default options. * See the overload for more details. * * @return The ListTopicsResult. */ public ListTopicsResult listTopics() { return listTopics(new ListTopicsOptions()); }
private void callAdminClientApisAndExpectAnAuthenticationError(AdminClientUnitTestEnv env) throws InterruptedException { try { env.adminClient().createTopics( Collections.singleton(new NewTopic("myTopic", Collections.singletonMap(0, asList(0, 1, 2)))), new CreateTopicsOptions().timeoutMs(10000)).all().get(); counts.put("my_topic", NewPartitions.increaseTo(3)); counts.put("other_topic", NewPartitions.increaseTo(3, asList(asList(2), asList(3)))); env.adminClient().createPartitions(counts).all().get(); fail("Expected an authentication error."); } catch (ExecutionException e) { env.adminClient().createAcls(asList(ACL1, ACL2)).all().get(); fail("Expected an authentication error."); } catch (ExecutionException e) { env.adminClient().describeAcls(FILTER1).values().get(); fail("Expected an authentication error."); } catch (ExecutionException e) { env.adminClient().deleteAcls(asList(FILTER1, FILTER2)).all().get(); fail("Expected an authentication error."); } catch (ExecutionException e) { env.adminClient().describeConfigs(Collections.singleton(new ConfigResource(ConfigResource.Type.BROKER, "0"))).all().get(); fail("Expected an authentication error."); } catch (ExecutionException e) {
@Override boolean runInternal(StopWatch stopWatch) throws InterruptedException, ExecutionException { stopWatch.start("adminClient.listTopics()"); Collection<String> topicNames = adminClient.listTopics().listings().get() .stream().map(TopicListing::name).filter(this::shouldCollectEvent).collect(Collectors.toList()); topicsMap.removeAll(new RemoveTopicPredicate(topicNames)); DescribeTopicsResult describeTopicsResult = adminClient.describeTopics(topicNames); describeTopicsResult.all().get().forEach( (topic, topicDescription) -> topicsMap.executeOnKey(topic, new SetTopicPartitionsProcessor( topicDescription.partitions().stream().map(TopicPartitionInfo::partition).collect(Collectors.toList())) ) ); metaMap.set(this.getName() + TopicServiceScheduler.LAST_SUCCESS_PREFIX, System.currentTimeMillis()); log.debug("Topics:" + topicsMap.entrySet()); log.debug(stopWatch.prettyPrint()); return true; }
/** * Get information about the nodes in the cluster, using the default options. * * This is a convenience method for #{@link AdminClient#describeCluster(DescribeClusterOptions)} with default options. * See the overload for more details. * * @return The DescribeClusterResult. */ public DescribeClusterResult describeCluster() { return describeCluster(new DescribeClusterOptions()); }