@Override public void start() { LOG.info("Starting embedded kafka server..."); kafkaServer = new EmbeddedKafkaServer(kafkaProperties); Service.State state = kafkaServer.startAndWait(); if (state != Service.State.RUNNING) { throw new IllegalStateException("Kafka server has not started... terminating."); } LOG.info("Embedded kafka server started successfully."); }
@Override protected void shutDown() throws Exception { // Flush all logs before shutting down Kafka server Loggings.forceFlush(); // Delay for 2 seconds to give clients chance to poll the last batch of log messages. try { TimeUnit.SECONDS.sleep(2); } catch (InterruptedException e) { // Ignore LOG.info("Kafka shutdown delay interrupted", e); } finally { kafkaServer.stopAndWait(); } }
@Override public void stop() { LOG.info("Stopping embedded kafka server..."); if (kafkaServer != null && kafkaServer.isRunning()) { kafkaServer.stopAndWait(); } }
private ApplicationKafkaService(ZKClient zkClient, String kafkaZKConnect) { this.zkClient = zkClient; this.kafkaServer = new EmbeddedKafkaServer(generateKafkaConfig(kafkaZKConnect)); this.kafkaZKPath = kafkaZKConnect.substring(zkClient.getConnectString().length()); }
@Override protected void startUp() throws Exception { // Create the ZK node for Kafka to use. If the node already exists, delete it to make sure there is // no left over content from previous AM attempt. LOG.info("Preparing Kafka ZK path {}{}", zkClient.getConnectString(), kafkaZKPath); ZKOperations.createDeleteIfExists(zkClient, kafkaZKPath, null, CreateMode.PERSISTENT, true).get(); kafkaServer.startAndWait(); }
@Override protected void startUp() throws Exception { int tries = 0; do { KafkaConfig kafkaConfig = createKafkaConfig(properties); KafkaServer kafkaServer = createKafkaServer(kafkaConfig); try { kafkaServer.startup(); server = kafkaServer; } catch (Exception e) { kafkaServer.shutdown(); kafkaServer.awaitShutdown(); Throwable rootCause = Throwables.getRootCause(e); if (rootCause instanceof ZkTimeoutException) { // Potentially caused by race condition bug described in TWILL-139. LOG.warn("Timeout when connecting to ZooKeeper from KafkaServer. Attempt number {}.", tries, rootCause); } else if (rootCause instanceof BindException) { LOG.warn("Kafka failed to bind to port {}. Attempt number {}.", kafkaConfig.port(), tries, rootCause); } else { throw e; } // Do a random sleep of < 200ms TimeUnit.MILLISECONDS.sleep(new Random().nextInt(200) + 1L); } } while (server == null && ++tries < startTimeoutRetries); if (server == null) { throw new IllegalStateException("Failed to start Kafka server after " + tries + " attempts."); } }
private ApplicationKafkaService(ZKClient zkClient, String kafkaZKConnect) { this.zkClient = zkClient; this.kafkaServer = new EmbeddedKafkaServer(generateKafkaConfig(kafkaZKConnect)); this.kafkaZKPath = kafkaZKConnect.substring(zkClient.getConnectString().length()); }
@Override protected void startUp() throws Exception { // Create the ZK node for Kafka to use. If the node already exists, delete it to make sure there is // no left over content from previous AM attempt. LOG.info("Preparing Kafka ZK path {}{}", zkClient.getConnectString(), kafkaZKPath); ZKOperations.createDeleteIfExists(zkClient, kafkaZKPath, null, CreateMode.PERSISTENT, true).get(); kafkaServer.startAndWait(); }
@Override protected void startUp() throws Exception { int tries = 0; do { KafkaConfig kafkaConfig = createKafkaConfig(properties); KafkaServer kafkaServer = createKafkaServer(kafkaConfig); try { kafkaServer.startup(); server = kafkaServer; } catch (Exception e) { kafkaServer.shutdown(); kafkaServer.awaitShutdown(); Throwable rootCause = Throwables.getRootCause(e); if (rootCause instanceof ZkTimeoutException) { // Potentially caused by race condition bug described in TWILL-139. LOG.warn("Timeout when connecting to ZooKeeper from KafkaServer. Attempt number {}.", tries, rootCause); } else if (rootCause instanceof BindException) { LOG.warn("Kafka failed to bind to port {}. Attempt number {}.", kafkaConfig.port(), tries, rootCause); } else { throw e; } // Do a random sleep of < 200ms TimeUnit.MILLISECONDS.sleep(new Random().nextInt(200) + 1L); } } while (server == null && ++tries < startTimeoutRetries); if (server == null) { throw new IllegalStateException("Failed to start Kafka server after " + tries + " attempts."); } }
@Override public void start() { LOG.info("Starting embedded kafka server..."); kafkaServer = new EmbeddedKafkaServer(kafkaProperties); Service.State state = kafkaServer.startAndWait(); if (state != Service.State.RUNNING) { throw new IllegalStateException("Kafka server has not started... terminating."); } LOG.info("Embedded kafka server started successfully."); }
@Override protected void shutDown() throws Exception { // Flush all logs before shutting down Kafka server Loggings.forceFlush(); // Delay for 2 seconds to give clients chance to poll the last batch of log messages. try { TimeUnit.SECONDS.sleep(2); } catch (InterruptedException e) { // Ignore LOG.info("Kafka shutdown delay interrupted", e); } finally { kafkaServer.stopAndWait(); } }
@Override public void stop() { LOG.info("Stopping embedded kafka server..."); if (kafkaServer != null && kafkaServer.isRunning()) { kafkaServer.stopAndWait(); } }
private EmbeddedKafkaServer createKafkaServer(String zkConnectStr, File dir) throws Exception { // Don't set port, EmbeddedKafkaServer will find a ephemeral port. Properties properties = new Properties(); properties.setProperty("broker.id", "1"); properties.setProperty("num.network.threads", "2"); properties.setProperty("num.io.threads", "2"); properties.setProperty("socket.send.buffer.bytes", "1048576"); properties.setProperty("socket.receive.buffer.bytes", "1048576"); properties.setProperty("socket.request.max.bytes", "104857600"); properties.setProperty("log.dir", dir.getAbsolutePath()); properties.setProperty("num.partitions", "1"); properties.setProperty("log.flush.interval.messages", "10000"); properties.setProperty("log.flush.interval.ms", "1000"); properties.setProperty("log.retention.hours", "1"); properties.setProperty("log.segment.bytes", "536870912"); properties.setProperty("log.cleanup.interval.mins", "1"); properties.setProperty("zookeeper.connect", zkConnectStr); properties.setProperty("zookeeper.connection.timeout.ms", "1000000"); return new EmbeddedKafkaServer(properties); } }
@Before public void beforeTest() throws Exception { zkServer = InMemoryZKServer.builder().setDataDir(TEMP_FOLDER.newFolder()).build(); zkServer.startAndWait(); CConfiguration cConf = CConfiguration.create(); String kafkaZKNamespace = cConf.get(KafkaConstants.ConfigKeys.ZOOKEEPER_NAMESPACE_CONFIG); kafkaZKConnect = zkServer.getConnectionStr(); if (kafkaZKNamespace != null) { ZKClientService zkClient = new DefaultZKClientService(zkServer.getConnectionStr(), 2000, null, ImmutableMultimap.<String, byte[]>of()); zkClient.startAndWait(); zkClient.create("/" + kafkaZKNamespace, null, CreateMode.PERSISTENT); zkClient.stopAndWait(); kafkaZKConnect += "/" + kafkaZKNamespace; } kafkaServer = createKafkaServer(kafkaZKConnect, TEMP_FOLDER.newFolder()); kafkaServer.startAndWait(); }
@Override protected void before() throws Throwable { tmpFolder.create(); zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build(); zkServer.startAndWait(); LOG.info("In memory ZK started on {}", zkServer.getConnectionStr()); kafkaServer = new EmbeddedKafkaServer(generateKafkaConfig()); kafkaServer.startAndWait(); initializeCConf(); injector = createInjector(); zkClient = injector.getInstance(ZKClientService.class); zkClient.startAndWait(); kafkaClient = injector.getInstance(KafkaClientService.class); kafkaClient.startAndWait(); brokerService = injector.getInstance(BrokerService.class); brokerService.startAndWait(); String brokerList = updateKafkaBrokerList(injector.getInstance(CConfiguration.class), brokerService); LOG.info("Waiting for Kafka server to startup..."); waitForKafkaStartup(); LOG.info("Kafka server started with broker list {}", brokerList); }
@After public void afterTest() { kafkaServer.stopAndWait(); zkServer.stopAndWait(); }
@Override protected void after() { brokerService.stopAndWait(); kafkaClient.stopAndWait(); zkClient.stopAndWait(); kafkaServer.stopAndWait(); zkServer.stopAndWait(); }