public void setUp() throws Exception { // setup ZK zookeeper = new TestingServer(true); // setup Broker kafkaDir = new TmpPath(Files.createTempDirectory("kafka-").toAbsolutePath().toString()); Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zookeeper.getConnectString()); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dirs", kafkaDir.getPath()); brokerProps.setProperty("listeners", String.format("PLAINTEXT://%s:%d", KAFKA_HOST, KAFKA_PORT)); brokerProps.setProperty("offsets.topic.replication.factor", "1"); KafkaConfig config = new KafkaConfig(brokerProps); MockTime mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); // setup default Producer createProducer(); kafkaAdminClient = AdminClient.create(Collections.singletonMap(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA_HOST + ":" + KAFKA_PORT)); }
private KafkaServer createKafkaServer(int brokerId,String _zkConnectString){ int _brokerId = brokerId; int _kafkaServerPort = TestUtils.findFreePort(); Properties props = kafka.utils.TestUtils.createBrokerConfig( _brokerId, _zkConnectString, kafka.utils.TestUtils.createBrokerConfig$default$3(), kafka.utils.TestUtils.createBrokerConfig$default$4(), _kafkaServerPort, kafka.utils.TestUtils.createBrokerConfig$default$6(), kafka.utils.TestUtils.createBrokerConfig$default$7(), kafka.utils.TestUtils.createBrokerConfig$default$8(), kafka.utils.TestUtils.createBrokerConfig$default$9(), kafka.utils.TestUtils.createBrokerConfig$default$10(), kafka.utils.TestUtils.createBrokerConfig$default$11(), kafka.utils.TestUtils.createBrokerConfig$default$12(), kafka.utils.TestUtils.createBrokerConfig$default$13(), kafka.utils.TestUtils.createBrokerConfig$default$14() ); KafkaConfig config = new KafkaConfig(props); Time mock = new MockTime(); KafkaServer _kafkaServer = kafka.utils.TestUtils.createServer(config, mock); kafkaBrokerPortList.add(_kafkaServerPort); return _kafkaServer; }
public void provisionTopic(String topic) { if (_topicConsumerMap.containsKey(topic)) { // nothing to do: return } else { // provision topic AdminUtils.createTopic(_kafkaServerSuite.getZkClient(), topic, 1, 1, new Properties()); List<KafkaServer> servers = new ArrayList<>(); servers.add(_kafkaServerSuite.getKafkaServer()); kafka.utils.TestUtils.waitUntilMetadataIsPropagated(scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 5000); KafkaConsumerSuite consumerSuite = new KafkaConsumerSuite(_kafkaServerSuite.getZkConnectString(), topic); _topicConsumerMap.put(topic, consumerSuite); } }
void start() throws RuntimeException { if (_numStarted.incrementAndGet() == 1) { log.warn("Starting up Kafka server suite. Zk at " + _zkConnectString + "; Kafka server at " + _kafkaServerPort); _zkServer = new EmbeddedZookeeper(_zkConnectString); _zkClient = new ZkClient(_zkConnectString, 30000, 30000, ZKStringSerializer$.MODULE$); Properties props = kafka.utils.TestUtils.createBrokerConfig(_brokerId, _kafkaServerPort, true); props.setProperty("zookeeper.connect", _zkConnectString); KafkaConfig config = new KafkaConfig(props); Time mock = new MockTime(); _kafkaServer = kafka.utils.TestUtils.createServer(config, mock); } else { log.info("Kafka server suite already started... continuing"); } }
public static void startServer() throws RuntimeException { if (serverStarted && serverClosed) { throw new RuntimeException("Kafka test server has already been closed. Cannot generate Kafka server twice."); } if (!serverStarted) { serverStarted = true; zkConnect = TestZKUtils.zookeeperConnect(); zkServer = new EmbeddedZookeeper(zkConnect); zkClient = new ZkClient(zkServer.connectString(), 30000, 30000, ZKStringSerializer$.MODULE$); kafkaPort = TestUtils.choosePort(); Properties props = TestUtils.createBrokerConfig(brokerId, kafkaPort, true); KafkaConfig config = new KafkaConfig(props); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); } }
Properties props = TestUtils.createBrokerConfig(0, TestUtils.choosePort(), false); KafkaConfig config = new KafkaConfig(props); KafkaServer kafkaServer = TestUtils.createServer(config, new MockTime()); TestUtils.createTopic(zkClient, topic, 1, 1, servers, new Properties()); zkClient.close(); TestUtils.waitUntilMetadataIsPropagated(servers, topic, 0, 5000);
this.brokerProperties.forEach(brokerConfigProperties::put); KafkaServer server = TestUtils.createServer(new KafkaConfig(brokerConfigProperties), Time.SYSTEM); this.kafkaServers.add(server); if (this.kafkaPorts[i] == 0) { this.kafkaPorts[i] = TestUtils.boundPort(server, SecurityProtocol.PLAINTEXT);
private Properties createBrokerProperties(int i) { return TestUtils.createBrokerConfig(i, this.zkConnect, this.controlledShutdown, true, this.kafkaPorts[i], scala.Option.apply(null), scala.Option.apply(null), scala.Option.apply(null), true, false, 0, false, 0, false, 0, scala.Option.apply(null), 1, false); }
.map(Integer::parseInt) .orElse(KAFKA_DEFAULT_PORT); Properties brokerConfigProperties = TestUtils.createBrokerConfig(0, this.zkConnect, this.controlledShutdown, true, kafkaPort, scala.Option.<SecurityProtocol>apply(null), brokerConfigProperties.setProperty("controller.socket.timeout.ms", "1000"); brokerConfigProperties.setProperty("offsets.topic.replication.factor", "1"); this.kafkaServer = TestUtils.createServer(new KafkaConfig(brokerConfigProperties), SystemTime$.MODULE$);
this.brokerProperties.forEach(brokerConfigProperties::put); KafkaServer server = TestUtils.createServer(new KafkaConfig(brokerConfigProperties), Time.SYSTEM); this.kafkaServers.add(server); if (this.kafkaPorts[i] == 0) { this.kafkaPorts[i] = TestUtils.boundPort(server, SecurityProtocol.PLAINTEXT);
private Properties createBrokerProperties(int i) { return TestUtils.createBrokerConfig(i, this.zkConnect, this.controlledShutdown, true, this.kafkaPorts[i], scala.Option.apply(null), scala.Option.apply(null), scala.Option.apply(null), true, false, 0, false, 0, false, 0, scala.Option.apply(null), 1, false); }
/** * Creates and starts an embedded Kafka broker. * * @param config Broker configuration settings. Used to modify, for example, the listeners * the broker should use. Note that you cannot change some settings such as * `log.dirs`. */ KafkaEmbedded(final Properties config) throws IOException { this.tmpFolder = new TemporaryFolder(); this.tmpFolder.create(); this.logDir = tmpFolder.newFolder(); this.effectiveConfig = effectiveConfigFrom(config, logDir); final KafkaConfig kafkaConfig = new KafkaConfig(effectiveConfig, true); log.debug("Starting embedded Kafka broker (with log.dirs={} and ZK ensemble at {}) ...", logDir, zookeeperConnect()); kafka = TestUtils.createServer(kafkaConfig, new SystemTime()); log.debug("Startup of embedded Kafka broker at {} completed (with ZK ensemble at {}) ...", brokerList(), zookeeperConnect()); }
_zkClient = new ZkClient(_zkConnectString, 30000, 30000, ZKStringSerializer$.MODULE$); Properties props = kafka.utils.TestUtils.createBrokerConfig( _brokerId, _zkConnectString, kafka.utils.TestUtils.createBrokerConfig$default$3(), kafka.utils.TestUtils.createBrokerConfig$default$4(), _kafkaServerPort, kafka.utils.TestUtils.createBrokerConfig$default$6(), kafka.utils.TestUtils.createBrokerConfig$default$7(), kafka.utils.TestUtils.createBrokerConfig$default$8(), kafka.utils.TestUtils.createBrokerConfig$default$9(), kafka.utils.TestUtils.createBrokerConfig$default$10(), kafka.utils.TestUtils.createBrokerConfig$default$11(), kafka.utils.TestUtils.createBrokerConfig$default$12(), kafka.utils.TestUtils.createBrokerConfig$default$13(), kafka.utils.TestUtils.createBrokerConfig$default$14() ); _kafkaServer = kafka.utils.TestUtils.createServer(config, mock);
Properties brokerConfigProperties = TestUtils.createBrokerConfig( nodeId, zkConnectString, enableControlledShutdown, true, port, scala.Option.apply(null), .setProperty("offsets.topic.replication.factor", "1"); brokerConfigProperties.put("zookeeper.connect", zkConnectString); kafkaServer = TestUtils.createServer( new KafkaConfig(brokerConfigProperties), Time.SYSTEM);
@Override public void before() throws IOException { this.zookeeper = new EmbeddedZookeeper(); String zkConnect = "127.0.0.1:" + zookeeper.port(); this.zkClient = new ZkClient(zkConnect, 5000, 5000, ZKStringSerializer$.MODULE$); for (int i = 0; i < numBrokers; i++) { ServerSocket ss = ServerSocketFactory.getDefault().createServerSocket(0); int brokerPort = ss.getLocalPort(); ss.close(); Properties props = TestUtils.createBrokerConfig(i, zkConnect, true, true, brokerPort, scala.Option.apply(null), scala.Option.apply(null), scala.Option.apply(null), true, false, 0, false, 0, false, 0, scala.Option.apply(null), 1, false); props.put(KafkaConfig.MinInSyncReplicasProp(), "1"); props.put(KafkaConfig.TransactionsTopicReplicationFactorProp(), "1"); props.put(KafkaConfig.TransactionsTopicMinISRProp(), "1"); this.brokers.add(new EmbeddedKafkaBroker(props)); } }
public KafkaTestBase(String topic) throws InterruptedException, RuntimeException { startServer(); this.topic = topic; AdminUtils.createTopic(zkClient, topic, 1, 1, new Properties()); List<KafkaServer> servers = new ArrayList<>(); servers.add(kafkaServer); TestUtils.waitUntilMetadataIsPropagated(scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 5000); Properties consumeProps = new Properties(); consumeProps.put("zookeeper.connect", zkConnect); consumeProps.put("group.id", "testConsumer"); consumeProps.put("zookeeper.session.timeout.ms", "10000"); consumeProps.put("zookeeper.sync.time.ms", "10000"); consumeProps.put("auto.commit.interval.ms", "10000"); consumeProps.put("consumer.timeout.ms", "10000"); consumer = Consumer.createJavaConsumerConnector(new ConsumerConfig(consumeProps)); Map<String, Integer> topicCountMap = new HashMap<>(); topicCountMap.put(this.topic, 1); Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap); List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(this.topic); stream = streams.get(0); iterator = stream.iterator(); }
/** * Override to set up your specific external resource. * * @throws Throwable if setup fails (which will disable {@code after} */ @Override protected void before() throws Throwable { // Start the ZK and the Broker LOG.info("init embedded Zookeeper"); zkServer = new EmbeddedZookeeper(); tmpLogDir = Files.createTempDirectory("kafka-log-dir-").toAbsolutePath(); String zkConnect = "127.0.0.1:" + zkServer.port(); LOG.info("init kafka broker"); Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dir", tmpLogDir.toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKER_IP_PORT); brokerProps.setProperty("offsets.topic.replication.factor", "1"); brokerProps.setProperty("transaction.state.log.replication.factor", "1"); brokerProps.setProperty("transaction.state.log.min.isr", "1"); KafkaConfig config = new KafkaConfig(brokerProps); kafkaServer = TestUtils.createServer(config, Time.SYSTEM); kafkaServer.startup(); kafkaServer.zkClient(); adminZkClient = new AdminZkClient(kafkaServer.zkClient()); LOG.info("Creating kafka TOPIC [{}]", TOPIC); adminZkClient.createTopic(TOPIC, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$); }
public void provisionTopic(String topic) { if (_topicConsumerMap.containsKey(topic)) { // nothing to do: return } else { // provision topic AdminUtils.createTopic(ZkUtils.apply(_kafkaServerSuite.getZkClient(), false), topic, 1, 1, new Properties()); List<KafkaServer> servers = new ArrayList<>(); servers.add(_kafkaServerSuite.getKafkaServer()); kafka.utils.TestUtils.waitUntilMetadataIsPropagated(scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 5000); KafkaConsumerSuite consumerSuite = new KafkaConsumerSuite(_kafkaServerSuite.getZkConnectString(), topic); _topicConsumerMap.put(topic, consumerSuite); } }
@Before public void setUp() throws IOException, SQLException { // setup Zookeeper zkServer = new EmbeddedZookeeper(); String zkConnect = ZKHOST + ":" + zkServer.port(); zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$); ZkUtils zkUtils = ZkUtils.apply(zkClient, false); // setup Broker Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT); KafkaConfig config = new KafkaConfig(brokerProps); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); kafkaServer.startup(); // create topic AdminUtils.createTopic(zkUtils, TOPIC, 1, 1, new Properties()); pConsumer = new PhoenixConsumer(); Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); conn = DriverManager.getConnection(getUrl(), props); }
public void createTopic(String topic) { AdminUtils.createTopic(zkUtils, topic, 1, 1, new Properties()); List<KafkaServer> servers = new ArrayList<KafkaServer>(); servers.add(kafkaServer); TestUtils.waitUntilMetadataIsPropagated(scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 30000); }