/** * Create a Kafka topic with the given parameters. * * @param topic The name of the topic. * @param partitions The number of partitions for this topic. * @param replication The replication factor for (partitions of) this topic. * @param topicConfig Additional topic-level configuration settings. */ void createTopic(final String topic, final int partitions, final int replication, final Map<String, String> topicConfig) { log.debug("Creating topic { name: {}, partitions: {}, replication: {}, config: {} }", topic, partitions, replication, topicConfig); final ImmutableMap<String, Object> props = ImmutableMap.of( AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList(), AdminClientConfig.RETRIES_CONFIG, 5); try (AdminClient adminClient = AdminClient.create(props)) { final NewTopic newTopic = new NewTopic(topic, partitions, (short) replication); newTopic.configs(topicConfig); try { final CreateTopicsResult result = adminClient.createTopics(ImmutableList.of(newTopic)); result.all().get(); } catch (final Exception e) { throw new RuntimeException("Failed to create topic:" + topic, e); } } }
@Test public void testInvalidTopicNames() throws Exception { try (AdminClientUnitTestEnv env = mockClientEnv()) { env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); List<String> sillyTopicNames = asList("", null); Map<String, KafkaFuture<Void>> deleteFutures = env.adminClient().deleteTopics(sillyTopicNames).values(); for (String sillyTopicName : sillyTopicNames) { TestUtils.assertFutureError(deleteFutures.get(sillyTopicName), InvalidTopicException.class); } assertEquals(0, env.kafkaClient().inFlightRequestCount()); Map<String, KafkaFuture<TopicDescription>> describeFutures = env.adminClient().describeTopics(sillyTopicNames).values(); for (String sillyTopicName : sillyTopicNames) { TestUtils.assertFutureError(describeFutures.get(sillyTopicName), InvalidTopicException.class); } assertEquals(0, env.kafkaClient().inFlightRequestCount()); List<NewTopic> newTopics = new ArrayList<>(); for (String sillyTopicName : sillyTopicNames) { newTopics.add(new NewTopic(sillyTopicName, 1, (short) 1)); } Map<String, KafkaFuture<Void>> createFutures = env.adminClient().createTopics(newTopics).values(); for (String sillyTopicName : sillyTopicNames) { TestUtils.assertFutureError(createFutures .get(sillyTopicName), InvalidTopicException.class); } assertEquals(0, env.kafkaClient().inFlightRequestCount()); } }
runnable.call(call, now); return new CreateTopicsResult(new HashMap<>(topicFutures));
public boolean manageTopic(Topic topic) { log.debug("Requested update for {}", topic.getName() ); if (topics().contains(topic.getName())) { updateTopic(topic); return true; } NewTopic nt = new NewTopic(topic.getName(), partitionCount(topic), replicationFactor(topic)); nt.configs(topic.getProperties()); CreateTopicsResult ct = adminClient.createTopics(singleton(nt)); try { ct.all().get(); createdTopics.inc(ct.values().size()); log.info("Created topic. name: {}, partitions: {}, replFactor: {}", topic.getName(), partitionCount(topic), replicationFactor(topic)); return true; } catch (InterruptedException | ExecutionException e) { if (e.getCause() != null && e.getCause() instanceof TopicExistsException) { log.debug("Topic exists. name {}", topic.getName()); updateTopic(topic); return true; } else { log.info("Exception occured during topic creation. name {}", topic.getName(), e); return false; } } }
public void createTopic(String topicName) throws Exception { kafkaAdminClient.createTopics(Collections.singleton(new NewTopic(topicName, 1, (short)1))) .all() .get(30, TimeUnit.SECONDS); }
private void createCruiseControlMetricsTopic() { try { final CreateTopicsResult createTopicsResult = _adminClient.createTopics(Collections.singletonList(_newTopic)); createTopicsResult.values().get(_newTopic.name()).get(); LOG.info("Cruise Control metrics topic created: {}", _cruiseControlMetricsTopic); } catch (InterruptedException | ExecutionException e) { if (!(e.getCause() instanceof TopicExistsException)) { LOG.error("Unable to create Cruise Control topic", e); } } }
return new CreateTopicsResult(createTopicResult); return new CreateTopicsResult(createTopicResult);
@Override public void createTestTopic(String topic, int numberOfPartitions, int replicationFactor, Properties properties) { LOG.info("Creating topic {}", topic); try (AdminClient adminClient = AdminClient.create(getStandardProperties())) { NewTopic topicObj = new NewTopic(topic, numberOfPartitions, (short) replicationFactor); adminClient.createTopics(Collections.singleton(topicObj)).all().get(); } catch (Exception e) { e.printStackTrace(); fail("Create test topic : " + topic + " failed, " + e.getMessage()); } }
/** * Create a new topic via the Kafka AdminClient API, calling the given handler * (in a different thread) with the result. */ @Override public void createTopic(Topic topic, Handler<AsyncResult<Void>> handler) { NewTopic newTopic = TopicSerialization.toNewTopic(topic, null); LOGGER.debug("Creating topic {}", newTopic); KafkaFuture<Void> future = adminClient.createTopics( Collections.singleton(newTopic)).values().get(newTopic.name()); queueWork(new UniWork<>("createTopic", future, handler)); }
@Test public void testCreateTopics() throws Exception { try (AdminClientUnitTestEnv env = mockClientEnv()) { env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); env.kafkaClient().prepareResponse(body -> body instanceof CreateTopicsRequest, new CreateTopicsResponse(Collections.singletonMap("myTopic", new ApiError(Errors.NONE, "")))); KafkaFuture<Void> future = env.adminClient().createTopics( Collections.singleton(new NewTopic("myTopic", Collections.singletonMap(0, asList(0, 1, 2)))), new CreateTopicsOptions().timeoutMs(10000)).all(); future.get(); } }
/** * Create new topic if not exists * * @param name topic name * @param partitions * @param rf replication factor * @param prop properties */ public static final void createTopic(final String name, final int partitions, final short rf, final Properties prop) { try (final AdminClient adminClient = KafkaAdminClient.create(prop)) { try { final NewTopic newTopic = new NewTopic(name, partitions, rf); final CreateTopicsResult createTopicsResult = adminClient.createTopics(Collections.singleton(newTopic)); createTopicsResult.values().get(name).get(); } catch (InterruptedException | ExecutionException e) { if (!(e.getCause() instanceof TopicExistsException)) { throw new RuntimeException(e.getMessage(), e); } // TopicExistsException - Swallow this exception, just means the topic already exists. } } }
/** * Test that the client properly times out when we don't receive any metadata. */ @Test public void testTimeoutWithoutMetadata() throws Exception { try (final AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(Time.SYSTEM, mockBootstrapCluster(), newStrMap(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, "10"))) { env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); env.kafkaClient().prepareResponse(new CreateTopicsResponse(Collections.singletonMap("myTopic", new ApiError(Errors.NONE, "")))); KafkaFuture<Void> future = env.adminClient().createTopics( Collections.singleton(new NewTopic("myTopic", Collections.singletonMap(0, asList(0, 1, 2)))), new CreateTopicsOptions().timeoutMs(1000)).all(); TestUtils.assertFutureError(future, TimeoutException.class); } }
/** * Creates a topic in Kafka. If the topic already exists this does nothing. * @param topicName the topic name to create. * @param partitions the number of partitions to create. * @param replicationFactor the number of replicas for the topic. */ public void createTopic(final String topicName, final int partitions, final short replicationFactor) { // Create admin client try (final AdminClient adminClient = getAdminClient()) { // Define topic final NewTopic newTopic = new NewTopic(topicName, partitions, replicationFactor); // Create topic, which is async call. final CreateTopicsResult createTopicsResult = adminClient.createTopics(Collections.singleton(newTopic)); // Since the call is Async, Lets wait for it to complete. createTopicsResult.values().get(topicName).get(); } catch (InterruptedException | ExecutionException e) { if (!(e.getCause() instanceof TopicExistsException)) { throw new RuntimeException(e.getMessage(), e); } // TopicExistsException - Swallow this exception, just means the topic already exists. } }
new CreateTopicsOptions().timeoutMs(10000)).all();
/** * Test that we propagate exceptions encountered when fetching metadata. */ @Test public void testPropagatedMetadataFetchException() throws Exception { Cluster cluster = mockCluster(0); try (final AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(Time.SYSTEM, cluster, newStrMap(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:8121", AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, "10"))) { env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); env.kafkaClient().createPendingAuthenticationError(cluster.nodeById(0), TimeUnit.DAYS.toMillis(1)); env.kafkaClient().prepareResponse(new CreateTopicsResponse(Collections.singletonMap("myTopic", new ApiError(Errors.NONE, "")))); KafkaFuture<Void> future = env.adminClient().createTopics( Collections.singleton(new NewTopic("myTopic", Collections.singletonMap(0, asList(0, 1, 2)))), new CreateTopicsOptions().timeoutMs(1000)).all(); TestUtils.assertFutureError(future, SaslAuthenticationException.class); } }
Map<String, KafkaFuture<Void>> newResults = admin.createTopics(topicsByName.values(), args).values();
@Test public void testUnreachableBootstrapServer() throws Exception { // This tests the scenario in which the bootstrap server is unreachable for a short while, // which prevents AdminClient from being able to send the initial metadata request Cluster cluster = Cluster.bootstrap(Collections.singletonList(new InetSocketAddress("localhost", 8121))); try (final AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(Time.SYSTEM, cluster)) { Cluster discoveredCluster = mockCluster(0); env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); env.kafkaClient().setUnreachable(cluster.nodes().get(0), 200); env.kafkaClient().prepareResponse(body -> body instanceof MetadataRequest, new MetadataResponse(discoveredCluster.nodes(), discoveredCluster.clusterResource().clusterId(), 1, Collections.emptyList())); env.kafkaClient().prepareResponse(body -> body instanceof CreateTopicsRequest, new CreateTopicsResponse(Collections.singletonMap("myTopic", new ApiError(Errors.NONE, "")))); KafkaFuture<Void> future = env.adminClient().createTopics( Collections.singleton(new NewTopic("myTopic", Collections.singletonMap(0, asList(0, 1, 2)))), new CreateTopicsOptions().timeoutMs(10000)).all(); future.get(); } }
for (final Map.Entry<String, KafkaFuture<Void>> createTopicResult : createTopicsResult.values().entrySet()) { try { createTopicResult.getValue().get();
env.adminClient().createTopics( Collections.singleton(new NewTopic("myTopic", Collections.singletonMap(0, asList(0, 1, 2)))), new CreateTopicsOptions().timeoutMs(10000)).all().get(); fail("Expected an authentication error."); } catch (ExecutionException e) {