/** * Create a Kafka topic with the given parameters. * * @param topic The name of the topic. * @param partitions The number of partitions for this topic. * @param replication The replication factor for (partitions of) this topic. * @param topicConfig Additional topic-level configuration settings. */ void createTopic(final String topic, final int partitions, final int replication, final Map<String, String> topicConfig) { log.debug("Creating topic { name: {}, partitions: {}, replication: {}, config: {} }", topic, partitions, replication, topicConfig); final ImmutableMap<String, Object> props = ImmutableMap.of( AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList(), AdminClientConfig.RETRIES_CONFIG, 5); try (AdminClient adminClient = AdminClient.create(props)) { final NewTopic newTopic = new NewTopic(topic, partitions, (short) replication); newTopic.configs(topicConfig); try { final CreateTopicsResult result = adminClient.createTopics(ImmutableList.of(newTopic)); result.all().get(); } catch (final Exception e) { throw new RuntimeException("Failed to create topic:" + topic, e); } } }
@Override public void initializeStorage() { super.initializeStorage(); try (AdminClient admin = AdminClient.create(this.producerConfig.asProperties())) { // Find default replication factor Config brokerConfig = getKafkaBrokerConfig(admin); final short replicationFactor = Short.parseShort(brokerConfig.get(DEFAULT_TOPIC_REPLICATION_FACTOR_PROP_NAME).value()); // Create topic final NewTopic topic = new NewTopic(topicName, (short)1, replicationFactor); topic.configs(Collect.hashMapOf("cleanup.policy", "delete", "retention.ms", Long.toString(Long.MAX_VALUE), "retention.bytes", "-1")); admin.createTopics(Collections.singleton(topic)); logger.info("Database history topic '{}' created", topic); } catch (Exception e) { throw new ConnectException("Creation of database history topic failed, please create the topic manually", e); } }
partitions.add(new TopicPartitionInfo(p, brokers.get(0), replicas, Collections.emptyList())); allTopics.put(topicName, new TopicMetadata(false, partitions, newTopic.configs())); future.complete(null); createTopicResult.put(topicName, future);
/** * Build the {@link NewTopic} representation. * * @return the topic description; never null */ public NewTopic build() { return new NewTopic(name, numPartitions, replicationFactor).configs(configs); } }
NewTopic newTopic() { NewTopic newTopic = new NewTopic(name, partitions, replicationFactor); newTopic.configs(configs); return newTopic; }
@SuppressWarnings({ "rawtypes", "unchecked" }) @Override public CompletableFuture<Void> createTopic(String topic, int partitions) { return CompletableFuture.runAsync(() -> { try { NewTopic newTopic = new NewTopic(topic, partitions, config.replicationFactor); newTopic.configs(new HashMap<>((Map) topicProperties)); admin.createTopics(Arrays.asList(newTopic)).all().get(); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } }); }
internalTopicConfig.numberOfPartitions(), replicationFactor) .configs(topicConfig));
configs.put(RETENTION_MS_CONFIG, "604800000"); // 7 day log retention for the messages configs.put(SEGMENT_MS_CONFIG, "604800000"); // roll over after 7 days even if not full topic.configs(configs); CreateTopicsResult result = adminClient.createTopics(Lists.newArrayList(topic)); configs.put(RETENTION_MS_CONFIG, "604800000"); // 7 day log retention for the messages configs.put(SEGMENT_MS_CONFIG, "604800000"); // roll over after 7 days even if not full topic.configs(configs); CreateTopicsResult result = adminClient.createTopics(Lists.newArrayList(topic)); configs.put(RETENTION_MS_CONFIG, ""+Long.MAX_VALUE); // forever log retention for the messages configs.put(SEGMENT_MS_CONFIG, "604800000"); // roll over after 7 days even if not full topic.configs(configs); CreateTopicsResult result = adminClient.createTopics(Lists.newArrayList(topic)); configs.put(RETENTION_MS_CONFIG, ""+Long.MAX_VALUE); // forever log retention for the messages configs.put(SEGMENT_MS_CONFIG, "604800000"); // roll over after 7 days even if not full topic.configs(configs); CreateTopicsResult result = adminClient.createTopics(Lists.newArrayList(topic)); configs.put(RETENTION_MS_CONFIG, ""+Long.MAX_VALUE); // forever log retention for the messages configs.put(SEGMENT_MS_CONFIG, "604800000"); // roll over after 7 days even if not full topic.configs(configs); CreateTopicsResult result = adminClient.createTopics(Lists.newArrayList(topic));
@Override public void initializeStorage() { super.initializeStorage(); try (AdminClient admin = AdminClient.create(this.producerConfig.asProperties())) { // Find default replication factor Config brokerConfig = getKafkaBrokerConfig(admin); final short replicationFactor = Short.parseShort(brokerConfig.get(DEFAULT_TOPIC_REPLICATION_FACTOR_PROP_NAME).value()); // Create topic final NewTopic topic = new NewTopic(topicName, (short)1, replicationFactor); topic.configs(Collect.hashMapOf("cleanup.policy", "delete", "retention.ms", Long.toString(Long.MAX_VALUE), "retention.bytes", "-1")); admin.createTopics(Collections.singleton(topic)); logger.info("Database history topic '{}' created", topic); } catch (Exception e) { throw new ConnectException("Creation of database history topic failed, please create the topic manually", e); } }
streamConfig.remove(REPL_FACTOR); newTopic.configs(new MapConfig(streamConfig)); CreateTopicsResult result = adminClient.createTopics(ImmutableSet.of(newTopic)); try {
/** * Create a NewTopic to reflect the given Topic. */ public static NewTopic toNewTopic(Topic topic, Map<Integer, List<Integer>> assignment) { NewTopic newTopic; if (assignment != null) { if (topic.getNumPartitions() != assignment.size()) { throw new IllegalArgumentException( format("Topic %s has %d partitions supplied, but the number of partitions " + "configured in KafkaTopic %s is %d", topic.getTopicName(), assignment.size(), topic.getResourceName(), topic.getNumPartitions())); } for (int partition = 0; partition < assignment.size(); partition++) { final List<Integer> value = assignment.get(partition); if (topic.getNumReplicas() != value.size()) { throw new IllegalArgumentException( format("Partition %d of topic %s has %d assigned replicas, " + "but the number of replicas configured in KafkaTopic %s for the topic is %d", partition, topic.getTopicName(), value.size(), topic.getResourceName(), topic.getNumReplicas())); } } newTopic = new NewTopic(topic.getTopicName().toString(), assignment); } else { newTopic = new NewTopic(topic.getTopicName().toString(), topic.getNumPartitions(), topic.getNumReplicas()); } newTopic.configs(topic.getConfig()); return newTopic; }
public boolean manageTopic(Topic topic) { log.debug("Requested update for {}", topic.getName() ); if (topics().contains(topic.getName())) { updateTopic(topic); return true; } NewTopic nt = new NewTopic(topic.getName(), partitionCount(topic), replicationFactor(topic)); nt.configs(topic.getProperties()); CreateTopicsResult ct = adminClient.createTopics(singleton(nt)); try { ct.all().get(); createdTopics.inc(ct.values().size()); log.info("Created topic. name: {}, partitions: {}, replFactor: {}", topic.getName(), partitionCount(topic), replicationFactor(topic)); return true; } catch (InterruptedException | ExecutionException e) { if (e.getCause() != null && e.getCause() instanceof TopicExistsException) { log.debug("Topic exists. name {}", topic.getName()); updateTopic(topic); return true; } else { log.info("Exception occured during topic creation. name {}", topic.getName(), e); return false; } } }
newTopic.configs(topicProperties.getProperties());
newTopic.configs(adminProperties.getConfiguration());