public void startCluster() { // Start Zookeeper. _zkServer = new EmbeddedZookeeper(); _zkConnectString = "127.0.0.1:"+_zkServer.port(); _zkClient = new ZkClient(_zkConnectString, 30000, 30000, ZKStringSerializer$.MODULE$); // Start Kafka Cluster. for(int i=0;i<clusterCount;i++) { KafkaServer _kafkaServer = createKafkaServer(i,_zkConnectString); kafkaBrokerList.add(_kafkaServer); } }
/** * Override to set up your specific external resource. * * @throws Throwable if setup fails (which will disable {@code after} */ @Override protected void before() throws Throwable { // Start the ZK and the Broker LOG.info("init embedded Zookeeper"); zkServer = new EmbeddedZookeeper(); tmpLogDir = Files.createTempDirectory("kafka-log-dir-").toAbsolutePath(); String zkConnect = "127.0.0.1:" + zkServer.port(); LOG.info("init kafka broker"); Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dir", tmpLogDir.toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKER_IP_PORT); brokerProps.setProperty("offsets.topic.replication.factor", "1"); brokerProps.setProperty("transaction.state.log.replication.factor", "1"); brokerProps.setProperty("transaction.state.log.min.isr", "1"); KafkaConfig config = new KafkaConfig(brokerProps); kafkaServer = TestUtils.createServer(config, Time.SYSTEM); kafkaServer.startup(); kafkaServer.zkClient(); adminZkClient = new AdminZkClient(kafkaServer.zkClient()); LOG.info("Creating kafka TOPIC [{}]", TOPIC); adminZkClient.createTopic(TOPIC, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$); }
void start() throws RuntimeException { if (_numStarted.incrementAndGet() == 1) { log.warn("Starting up Kafka server suite. Zk at " + _zkConnectString + "; Kafka server at " + _kafkaServerPort); _zkServer = new EmbeddedZookeeper(_zkConnectString); _zkClient = new ZkClient(_zkConnectString, 30000, 30000, ZKStringSerializer$.MODULE$); Properties props = kafka.utils.TestUtils.createBrokerConfig(_brokerId, _kafkaServerPort, true); props.setProperty("zookeeper.connect", _zkConnectString); KafkaConfig config = new KafkaConfig(props); Time mock = new MockTime(); _kafkaServer = kafka.utils.TestUtils.createServer(config, mock); } else { log.info("Kafka server suite already started... continuing"); } }
public static void startServer() throws RuntimeException { if (serverStarted && serverClosed) { throw new RuntimeException("Kafka test server has already been closed. Cannot generate Kafka server twice."); } if (!serverStarted) { serverStarted = true; zkConnect = TestZKUtils.zookeeperConnect(); zkServer = new EmbeddedZookeeper(zkConnect); zkClient = new ZkClient(zkServer.connectString(), 30000, 30000, ZKStringSerializer$.MODULE$); kafkaPort = TestUtils.choosePort(); Properties props = TestUtils.createBrokerConfig(brokerId, kafkaPort, true); KafkaConfig config = new KafkaConfig(props); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); } }
if (_numStarted.incrementAndGet() == 1) { log.warn("Starting up Kafka server suite. Zk at " + _zkConnectString + "; Kafka server at " + _kafkaServerPort); _zkServer = new EmbeddedZookeeper(); _zkConnectString = "127.0.0.1:"+_zkServer.port(); _zkClient = new ZkClient(_zkConnectString, 30000, 30000, ZKStringSerializer$.MODULE$);
@Override public void afterPropertiesSet() { this.zookeeper = new EmbeddedZookeeper(); int zkConnectionTimeout = 6000; // NOSONAR magic # int zkSessionTimeout = 6000; // NOSONAR magic #
@Before public void setUp() throws IOException, SQLException { // setup Zookeeper zkServer = new EmbeddedZookeeper(); String zkConnect = ZKHOST + ":" + zkServer.port(); zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$); ZkUtils zkUtils = ZkUtils.apply(zkClient, false); // setup Broker Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT); KafkaConfig config = new KafkaConfig(brokerProps); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); kafkaServer.startup(); // create topic AdminUtils.createTopic(zkUtils, TOPIC, 1, 1, new Properties()); pConsumer = new PhoenixConsumer(); Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); conn = DriverManager.getConnection(getUrl(), props); }
private void createKafkaCluster() throws IOException { System.setProperty("zookeeper.preAllocSize", Integer.toString(128)); zkServer = new EmbeddedZookeeper(); String zkConnect = ZK_HOST + ':' + zkServer.port(); ZkClient zkClient = new ZkClient(zkConnect, SESSION_TIMEOUT, CONNECTION_TIMEOUT, ZKStringSerializer$.MODULE$); zkUtils = ZkUtils.apply(zkClient, false); brokerPort = getRandomPort(); Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKER_HOST + ':' + brokerPort); brokerProps.setProperty("offsets.topic.replication.factor", "1"); brokerProps.setProperty("offsets.topic.num.partitions", "1"); KafkaConfig config = new KafkaConfig(brokerProps); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); }
/** * Starts the Embedded Kafka and Zookeeper Servers. * @throws Exception - If an exeption occurs during startup. */ protected void startup() throws Exception { // Setup the embedded zookeeper logger.info("Starting up Embedded Zookeeper..."); zkServer = new EmbeddedZookeeper(); zookeperConnect = ZKHOST + ":" + zkServer.port(); logger.info("Embedded Zookeeper started at: {}", zookeperConnect); // setup Broker logger.info("Starting up Embedded Kafka..."); brokerPort = Integer.toString(PortUtils.getRandomFreePort()); final Properties brokerProps = new Properties(); brokerProps.setProperty(KafkaConfig$.MODULE$.BrokerIdProp(), "0"); brokerProps.setProperty(KafkaConfig$.MODULE$.HostNameProp(), BROKERHOST); brokerProps.setProperty(KafkaConfig$.MODULE$.PortProp(), brokerPort); brokerProps.setProperty(KafkaConfig$.MODULE$.ZkConnectProp(), zookeperConnect); brokerProps.setProperty(KafkaConfig$.MODULE$.LogDirsProp(), Files.createTempDirectory(getClass().getSimpleName() + "-").toAbsolutePath().toString()); final KafkaConfig config = new KafkaConfig(brokerProps); final Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); logger.info("Embedded Kafka Server started at: {}:{}", BROKERHOST, brokerPort); }
/** * Starts the Embedded Kafka and Zookeeper Servers. * @throws Exception - If an exeption occurs during startup. */ protected void startup() throws Exception { // Setup the embedded zookeeper logger.info("Starting up Embedded Zookeeper..."); zkServer = new EmbeddedZookeeper(); zookeperConnect = ZKHOST + ":" + zkServer.port(); logger.info("Embedded Zookeeper started at: {}", zookeperConnect); // setup Broker logger.info("Starting up Embedded Kafka..."); brokerPort = Integer.toString(PortUtils.getRandomFreePort()); final Properties brokerProps = new Properties(); brokerProps.setProperty(KafkaConfig$.MODULE$.BrokerIdProp(), "0"); brokerProps.setProperty(KafkaConfig$.MODULE$.HostNameProp(), BROKERHOST); brokerProps.setProperty(KafkaConfig$.MODULE$.PortProp(), brokerPort); brokerProps.setProperty(KafkaConfig$.MODULE$.ZkConnectProp(), zookeperConnect); brokerProps.setProperty(KafkaConfig$.MODULE$.LogDirsProp(), Files.createTempDirectory(getClass().getSimpleName() + "-").toAbsolutePath().toString()); brokerProps.setProperty(KafkaConfig$.MODULE$.DeleteTopicEnableProp(), "true"); final KafkaConfig config = new KafkaConfig(brokerProps); final Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); logger.info("Embedded Kafka Server started at: {}:{}", BROKERHOST, brokerPort); }
/** * setup mini kafka and call the super to setup mini fluo */ @Before public void setupKafka() throws Exception { // Install an instance of Rya on the Accumulo cluster. installRyaInstance(); // Setup Kafka. zkServer = new EmbeddedZookeeper(); final String zkConnect = ZKHOST + ":" + zkServer.port(); zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$); zkUtils = ZkUtils.apply(zkClient, false); // setup Broker final Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT); final KafkaConfig config = new KafkaConfig(brokerProps); final Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); }
/** * setup mini kafka and call the super to setup mini fluo */ @Before public void setupKafka() throws Exception { // Install an instance of Rya on the Accumulo cluster. installRyaInstance(); // Setup Kafka. zkServer = new EmbeddedZookeeper(); final String zkConnect = ZKHOST + ":" + zkServer.port(); zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$); zkUtils = ZkUtils.apply(zkClient, false); // setup Broker final Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT); final KafkaConfig config = new KafkaConfig(brokerProps); final Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); }
private void createKafkaCluster() throws IOException { zkServer = new EmbeddedZookeeper(); String zkConnect = "localhost:" + zkServer.port(); ZkClient zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$); zkUtils = ZkUtils.apply(zkClient, false); KafkaConfig config = new KafkaConfig(props( "zookeeper.connect", zkConnect, "broker.id", "0", "log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString(), "offsets.topic.replication.factor", "1", "listeners", "PLAINTEXT://localhost:9092")); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); }
@Override public void afterPropertiesSet() { this.zookeeper = new EmbeddedZookeeper(); int zkConnectionTimeout = 6000; // NOSONAR magic # int zkSessionTimeout = 6000; // NOSONAR magic #
public void start() throws IOException { // Find port try { ServerSocket serverSocket = new ServerSocket(0); BROKERPORT = Integer.toString(serverSocket.getLocalPort()); serverSocket.close(); } catch (IOException e) { throw Throwables.propagate(e); } // Setup Zookeeper zkServer = new EmbeddedZookeeper(); String zkConnect = BROKERHOST + ":" + zkServer.port(); zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$); zkUtils = ZkUtils.apply(zkClient, false); // Setup brokers cleanupDir(); Properties props = new Properties(); props.setProperty("zookeeper.connect", zkConnect); props.setProperty("broker.id", "0"); props.setProperty("log.dirs", KAFKA_PATH); props.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT); KafkaConfig config = new KafkaConfig(props); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); }
@Override public void before() throws IOException { this.zookeeper = new EmbeddedZookeeper(); String zkConnect = "127.0.0.1:" + zookeeper.port(); this.zkClient = new ZkClient(zkConnect, 5000, 5000, ZKStringSerializer$.MODULE$); for (int i = 0; i < numBrokers; i++) { ServerSocket ss = ServerSocketFactory.getDefault().createServerSocket(0); int brokerPort = ss.getLocalPort(); ss.close(); Properties props = TestUtils.createBrokerConfig(i, zkConnect, true, true, brokerPort, scala.Option.apply(null), scala.Option.apply(null), scala.Option.apply(null), true, false, 0, false, 0, false, 0, scala.Option.apply(null), 1, false); props.put(KafkaConfig.MinInSyncReplicasProp(), "1"); props.put(KafkaConfig.TransactionsTopicReplicationFactorProp(), "1"); props.put(KafkaConfig.TransactionsTopicMinISRProp(), "1"); this.brokers.add(new EmbeddedKafkaBroker(props)); } }
EmbeddedZookeeper zkServer = new EmbeddedZookeeper(TestZKUtils.zookeeperConnect()); ZkClient zkClient = new ZkClient(zkServer.connectString(), 30000, 30000, ZKStringSerializer$.MODULE$); Properties props = TestUtils.createBrokerConfig(0, TestUtils.choosePort(), false);