public static KafkaServerStartable startServer(final int port, final int brokerId, final String zkStr, final Properties configuration) { // Create the ZK nodes for Kafka, if needed int indexOfFirstSlash = zkStr.indexOf('/'); if (indexOfFirstSlash != -1) { String bareZkUrl = zkStr.substring(0, indexOfFirstSlash); String zkNodePath = zkStr.substring(indexOfFirstSlash); ZkClient client = new ZkClient(bareZkUrl); client.createPersistent(zkNodePath, true); client.close(); } File logDir = new File("/tmp/kafka-" + Double.toHexString(Math.random())); logDir.mkdirs(); configureKafkaPort(configuration, port); configureZkConnectionString(configuration, zkStr); configureBrokerId(configuration, brokerId); configureKafkaLogDirectory(configuration, logDir); KafkaConfig config = new KafkaConfig(configuration); KafkaServerStartable serverStartable = new KafkaServerStartable(config); serverStartable.startup(); return serverStartable; }
@BeforeTest public void setup() { LOGGER.info("Trying to setup"); ZkStarter.startLocalZkServer(); kafkaStarter = KafkaStarterUtils.startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration()); // Create Kafka topic KafkaStarterUtils.createTopic("testTopic0", KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(2000); } catch (Exception e) { } kafkaBrokerTopicObserver = new KafkaBrokerTopicObserver("broker0", KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(3000); } catch (Exception e) { } }
@AfterTest public void shutdown() { LOGGER.info("Trying to shutdown"); kafkaBrokerTopicObserver.stop(); KafkaStarterUtils.stopServer(kafkaStarter); ZkStarter.stopLocalZkServer(); }
@BeforeTest public void setup() { LOGGER.info("Trying to setup"); ZkStarter.startLocalZkServer(); kafkaStarter = KafkaStarterUtils.startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration()); try { Thread.sleep(2000); } catch (Exception e) { } kafkaBrokerTopicObserver = new KafkaBrokerTopicObserver("broker0", KafkaStarterUtils.DEFAULT_ZK_STR, 1); ZK_CLIENT = new ZkClient(ZkStarter.DEFAULT_ZK_STR); ZK_CLIENT.deleteRecursive("/" + HELIX_CLUSTER_NAME); REQUEST_URL = "http://localhost:" + CONTROLLER_PORT; CONTROLLER_STARTER = startController(DEPLOYMENT_NAME, HELIX_CLUSTER_NAME, CONTROLLER_PORT); try { FAKE_INSTANCES.addAll(ControllerTestUtils .addFakeDataInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZkStarter.DEFAULT_ZK_STR, 4, 0)); Thread.sleep(4000); } catch (Exception e) { throw new RuntimeException("Error during adding fake instances"); } }
@Test public void testAutoTopic() { for (int i = 0; i < 10; ++i) { Assert.assertEquals(helixMirrorMakerManager.getTopicLists().size(), i); String topicName = "testTopic" + i; // Create Kafka topic KafkaStarterUtils.createTopic(topicName, KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(5000); } catch (Exception e) { } Assert.assertEquals(kafkaBrokerTopicObserver.getNumTopics(), 1 + i); for (int j = 0; j <= i; ++j) { Assert.assertTrue(kafkaBrokerTopicObserver.getAllTopics().contains("testTopic" + j)); Assert.assertEquals( kafkaBrokerTopicObserver.getTopicPartition("testTopic" + j).getPartition(), 1); } } } }
@BeforeTest public void setup() { LOGGER.info("Trying to setup"); ZkStarter.startLocalZkServer(); kafkaStarter = KafkaStarterUtils.startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration()); try { Thread.sleep(2000); } catch (Exception e) { } kafkaBrokerTopicObserver = new KafkaBrokerTopicObserver("broker0", KafkaStarterUtils.DEFAULT_ZK_STR, 1); ControllerConf controllerConf = new ControllerConf(); controllerConf.setControllerPort("9090"); controllerConf.setHelixClusterName("TestAutoTopicWhitelistingManager"); controllerConf.setDeploymentName("TestAutoTopicWhitelistingManagerDeployment"); controllerConf.setInstanceId("controller-0"); controllerConf.setControllerMode("customized"); controllerConf.setZkStr(ZkStarter.DEFAULT_ZK_STR); controllerConf.setBackUpToGit("false"); controllerConf.setAutoRebalanceDelayInSeconds("1"); helixMirrorMakerManager = new HelixMirrorMakerManager(controllerConf); helixMirrorMakerManager.start(); sourceKafkaClusterValidationManager = new SourceKafkaClusterValidationManager(kafkaBrokerTopicObserver, helixMirrorMakerManager); sourceKafkaClusterValidationManager.start(); }
@Test public void testKafkaBrokerTopicObserver() { Assert.assertEquals(kafkaBrokerTopicObserver.getNumTopics(), 1); Assert.assertEquals(kafkaBrokerTopicObserver.getTopicPartition("testTopic0").getPartition(), 1); for (int i = 1; i < 10; ++i) { String topicName = "testTopic" + i; // Create Kafka topic KafkaStarterUtils.createTopic(topicName, KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(5000); } catch (Exception e) { } Assert.assertEquals(kafkaBrokerTopicObserver.getNumTopics(), 1 + i); for (int j = 0; j <= i; ++j) { Assert.assertTrue(kafkaBrokerTopicObserver.getAllTopics().contains("testTopic" + j)); Assert.assertEquals( kafkaBrokerTopicObserver.getTopicPartition("testTopic" + j).getPartition(), 1); } } } }
ZkStarter.startLocalZkServer(); kafkaStarter = KafkaStarterUtils.startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration()); KafkaStarterUtils.createTopic("testTopic0", KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(2000); KafkaStarterUtils.createTopic("testTopic0", KafkaStarterUtils.DEFAULT_ZK_STR);
public static KafkaServerStartable startServer(final int port, final int brokerId, final String zkStr, final Properties configuration) { // Create the ZK nodes for Kafka, if needed int indexOfFirstSlash = zkStr.indexOf('/'); if (indexOfFirstSlash != -1) { String bareZkUrl = zkStr.substring(0, indexOfFirstSlash); String zkNodePath = zkStr.substring(indexOfFirstSlash); ZkClient client = new ZkClient(bareZkUrl); client.createPersistent(zkNodePath, true); client.close(); } File logDir = new File("/tmp/kafka-" + Double.toHexString(Math.random())); logDir.mkdirs(); logDir.deleteOnExit(); configureKafkaPort(configuration, port); configureZkConnectionString(configuration, zkStr); configureBrokerId(configuration, brokerId); configureKafkaLogDirectory(configuration, logDir); KafkaConfig config = new KafkaConfig(configuration); KafkaServerStartable serverStartable = new KafkaServerStartable(config); serverStartable.startup(); return serverStartable; }
@BeforeTest public void setup() { LOGGER.info("Trying to setup"); ZkStarter.startLocalZkServer(); kafkaStarter = KafkaStarterUtils.startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration()); try { Thread.sleep(2000); } catch (Exception e) { } kafkaBrokerTopicObserver = new KafkaBrokerTopicObserver("broker0", KafkaStarterUtils.DEFAULT_ZK_STR, 1); ControllerConf controllerConf = new ControllerConf(); controllerConf.setControllerPort("9090"); controllerConf.setHelixClusterName("TestAutoTopicWhitelistingManager"); controllerConf.setDeploymentName("TestAutoTopicWhitelistingDeployment"); controllerConf.setInstanceId("controller-0"); controllerConf.setControllerMode("customized"); controllerConf.setZkStr(ZkStarter.DEFAULT_ZK_STR); controllerConf.setBackUpToGit("false"); controllerConf.setAutoRebalanceDelayInSeconds("1"); helixMirrorMakerManager = new HelixMirrorMakerManager(controllerConf); helixMirrorMakerManager.start(); autoTopicWhitelistingManager = new AutoTopicWhitelistingManager(kafkaBrokerTopicObserver, kafkaBrokerTopicObserver, helixMirrorMakerManager, "", 1); autoTopicWhitelistingManager.start(); }
@AfterTest public void shutdown() { LOGGER.info("Trying to shutdown"); helixMirrorMakerManager.stop(); kafkaBrokerTopicObserver.stop(); KafkaStarterUtils.stopServer(kafkaStarter); ZkStarter.stopLocalZkServer(); }
String topicName = "testTopic" + i; KafkaStarterUtils.createTopic(topicName, KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(5000); String topicName = "testTopic" + i; KafkaStarterUtils.createTopic(topicName, KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(5000); String topicName = "testTopic" + i; KafkaStarterUtils.createTopic(topicName, KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(5000);
ZkStarter.startLocalZkServer(); kafkaStarter = KafkaStarterUtils.startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration()); KafkaStarterUtils.createTopic("testTopic0", KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(2000); KafkaStarterUtils.createTopic("testTopic0", KafkaStarterUtils.DEFAULT_ZK_STR);
@AfterTest public void shutdown() { LOGGER.info("Trying to shutdown"); helixMirrorMakerManager.stop(); kafkaBrokerTopicObserver.stop(); KafkaStarterUtils.stopServer(kafkaStarter); ZkStarter.stopLocalZkServer(); }
@Test public void testAutoTopic() { for (int i = 0; i < 10; ++i) { Assert.assertEquals(helixMirrorMakerManager.getTopicLists().size(), i); String topicName = "testTopic" + i; // Create Kafka topic KafkaStarterUtils.createTopic(topicName, KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(5000); } catch (Exception e) { } Assert.assertEquals(kafkaBrokerTopicObserver.getNumTopics(), 1 + i); for (int j = 0; j <= i; ++j) { Assert.assertTrue(kafkaBrokerTopicObserver.getAllTopics().contains("testTopic" + j)); Assert.assertEquals( kafkaBrokerTopicObserver.getTopicPartition("testTopic" + j).getPartition(), 1); } } } }
ZkStarter.startLocalZkServer(); kafkaStarter = KafkaStarterUtils.startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration()); KafkaStarterUtils.createTopic("testTopic0", KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(2000);
@AfterTest public void shutdown() { LOGGER.info("Trying to shutdown"); helixMirrorMakerManager.stop(); kafkaBrokerTopicObserver.stop(); KafkaStarterUtils.stopServer(kafkaStarter); ZkStarter.stopLocalZkServer(); }
@Test public void testKafkaBrokerTopicObserver() { Assert.assertEquals(kafkaBrokerTopicObserver.getNumTopics(), 1); Assert.assertEquals(kafkaBrokerTopicObserver.getTopicPartition("testTopic0").getPartition(), 1); for (int i = 1; i < 10; ++i) { String topicName = "testTopic" + i; // Create Kafka topic KafkaStarterUtils.createTopic(topicName, KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(5000); } catch (Exception e) { } Assert.assertEquals(kafkaBrokerTopicObserver.getNumTopics(), 1 + i); for (int j = 0; j <= i; ++j) { Assert.assertTrue(kafkaBrokerTopicObserver.getAllTopics().contains("testTopic" + j)); Assert.assertEquals( kafkaBrokerTopicObserver.getTopicPartition("testTopic" + j).getPartition(), 1); } } } }
@BeforeTest public void setup() { LOGGER.info("Trying to setup"); ZkStarter.startLocalZkServer(); kafkaStarter = KafkaStarterUtils.startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration()); // Create Kafka topic KafkaStarterUtils.createTopic("testTopic0", KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(2000); } catch (Exception e) { } kafkaBrokerTopicObserver = new KafkaBrokerTopicObserver("broker0", KafkaStarterUtils.DEFAULT_ZK_STR); ControllerConf controllerConf = new ControllerConf(); controllerConf.setControllerPort("9090"); controllerConf.setHelixClusterName("TestAutoTopicWhitelistingManager"); controllerConf.setInstanceId("controller-0"); controllerConf.setControllerMode("customized"); controllerConf.setZkStr(ZkStarter.DEFAULT_ZK_STR); controllerConf.setBackUpToGit("false"); controllerConf.setAutoRebalanceDelayInSeconds("1"); helixMirrorMakerManager = new HelixMirrorMakerManager(controllerConf); helixMirrorMakerManager.start(); sourceKafkaClusterValidationManager = new SourceKafkaClusterValidationManager(kafkaBrokerTopicObserver, helixMirrorMakerManager); sourceKafkaClusterValidationManager.start(); }
@AfterTest public void shutdown() { LOGGER.info("Trying to shutdown"); for (FakeInstance fakeInstance : FAKE_INSTANCES) { try { LOGGER.info("Trying to shutdown: " + fakeInstance); fakeInstance.stop(); } catch (Exception e) { } } LOGGER.info("Trying to stop controller"); CONTROLLER_STARTER.stop(); LOGGER.info("Trying to stop zk"); kafkaBrokerTopicObserver.stop(); KafkaStarterUtils.stopServer(kafkaStarter); ZK_CLIENT.deleteRecursive("/" + HELIX_CLUSTER_NAME); ZK_CLIENT.close(); ZkStarter.stopLocalZkServer(); }