public static void uploadConfigsToZookeeper(String globalConfigPath, String parsersConfigPath, String enrichmentsConfigPath, String indexingConfigPath, String profilerConfigPath, String zookeeperUrl) throws Exception { try (CuratorFramework client = getClient(zookeeperUrl)) { client.start(); uploadConfigsToZookeeper(globalConfigPath, parsersConfigPath, enrichmentsConfigPath, indexingConfigPath, profilerConfigPath, client); } }
public static void writeGlobalConfigToZookeeper(byte[] globalConfig, String zookeeperUrl) throws Exception { try(CuratorFramework client = getClient(zookeeperUrl)) { client.start(); writeGlobalConfigToZookeeper(globalConfig, client); } }
public static void writeGlobalConfigToZookeeper(Map<String, Object> globalConfig, String zookeeperUrl) throws Exception { try(CuratorFramework client = getClient(zookeeperUrl)) { client.start(); writeGlobalConfigToZookeeper(globalConfig, client); } } public static void writeGlobalConfigToZookeeper(Map<String, Object> globalConfig, CuratorFramework client) throws Exception {
public static void writeSensorEnrichmentConfigToZookeeper(String sensorType, byte[] configData, String zookeeperUrl) throws Exception { try(CuratorFramework client = getClient(zookeeperUrl)) { client.start(); writeSensorEnrichmentConfigToZookeeper(sensorType, configData, client); } }
public static byte[] readFromZookeeper(String path, String zookeeperUrl) throws Exception { try (CuratorFramework client = getClient(zookeeperUrl)) { client.start(); return readFromZookeeper(path, client); } }
/** * Creates a Zookeeper client if it doesn't exist and a url for zk is provided. * @param zookeeperUrl The Zookeeper URL. */ private Optional<CuratorFramework> setupClient(Optional<CuratorFramework> zkClient, String zookeeperUrl) { // can only create client if we have a valid zookeeper URL if (!zkClient.isPresent()) { if (StringUtils.isNotBlank(zookeeperUrl)) { CuratorFramework client = ConfigurationsUtils.getClient(zookeeperUrl); client.start(); return Optional.of(client); } else { LOG.warn("Unable to setup zookeeper client - zk_quorum url not provided. **This will limit some Stellar functionality**"); return Optional.empty(); } } else { return zkClient; } }
public static void writeSensorParserConfigToZookeeper(String sensorType, byte[] configData, String zookeeperUrl) throws Exception { try(CuratorFramework client = getClient(zookeeperUrl)) { client.start(); writeSensorParserConfigToZookeeper(sensorType, configData, client); } }
public static void writeSensorIndexingConfigToZookeeper(String sensorType, byte[] configData, String zookeeperUrl) throws Exception { try(CuratorFramework client = getClient(zookeeperUrl)) { client.start(); writeSensorIndexingConfigToZookeeper(sensorType, configData, client); } }
/** * Writes config to path in Zookeeper, /metron/topology/$CONFIG_TYPE/$CONFIG_NAME */ public static void writeConfigToZookeeper(String configPath, byte[] config, String zookeeperUrl) throws Exception { try (CuratorFramework client = getClient(zookeeperUrl)) { client.start(); writeToZookeeper(configPath, config, client); } }
/** * Reads Json data for the specified config type and config name (if applicable) from zookeeper, * applies the patch from patchData, and writes it back to Zookeeper in a pretty print format. * Patching JSON flattens existing formatting, so this will keep configs readable. * Starts up curatorclient based on zookeeperUrl. * * @param configurationType GLOBAL, PARSER, etc. * @param configName e.g. bro, yaf, snort * @param patchData a JSON patch in the format specified by RFC 6902 * @param zookeeperUrl configs are here */ public static void applyConfigPatchToZookeeper(ConfigurationType configurationType, Optional<String> configName, byte[] patchData, String zookeeperUrl) throws Exception { try (CuratorFramework client = getClient(zookeeperUrl)) { client.start(); applyConfigPatchToZookeeper(configurationType, configName, patchData, client); } }
public void run(CommandLine cli) throws Exception { try(CuratorFramework client = ConfigurationsUtils.getClient(ConfigurationOptions.ZK_QUORUM.get(cli))) { client.start(); run(client, cli); } } public void run(CuratorFramework client, CommandLine cli) throws Exception {
protected void pushConfig(Path srcPath, Path dstPath, String configName, String zookeeper) { System.out.println("Beginning update of global configs"); try (CuratorFramework client = ConfigurationsUtils.getClient(zookeeper)) { client.start(); // Use the parent and place a new file. Has to be a new file so we can update the configs and trigger updates. // Fetch the global configuration Map<String, Object> global = JSONUtils.INSTANCE.load( new ByteArrayInputStream(ConfigurationsUtils.readGlobalConfigBytesFromZookeeper(client)), JSONUtils.MAP_SUPPLIER); // Update the global config and push it back global.put(configName, dstPath.toString() + "/" + srcPath.getName()); ConfigurationsUtils.writeGlobalConfigToZookeeper(global, client); } catch (Exception e) { System.err.println("Unable to load new GeoLite2 config for " + configName + " into HDFS: " + e); e.printStackTrace(); System.exit(2); } System.out.println("Finished update of global configs"); }
public void updateSensorConfigs( ) throws Exception { CuratorFramework client = ConfigurationsUtils.getClient(getZkQuorum()); try { client.start(); updateSensorConfigs(new ZKSourceConfigHandler(client), sensorToFieldList); } finally { client.close(); } }
/** * Fetch the parser configuration from Zookeeper. * * @param zookeeperUrl Zookeeper URL * @param sensorTypes Types of sensor * @param configs * @return * @throws Exception */ private static Map<String, SensorParserConfig> getSensorParserConfig(String zookeeperUrl, List<String> sensorTypes, ParserConfigurations configs) throws Exception { Map<String, SensorParserConfig> parserConfigs = new HashMap<>(); try(CuratorFramework client = ConfigurationsUtils.getClient(zookeeperUrl)) { client.start(); ConfigurationsUtils.updateParserConfigsFromZookeeper(configs, client); for (String sensorType : sensorTypes) { SensorParserConfig parserConfig = configs.getSensorParserConfig(sensorType); if (parserConfig == null) { throw new IllegalStateException("Cannot find the parser configuration in zookeeper for " + sensorType + "." + " Please check that it exists in zookeeper by using the 'zk_load_configs.sh -m DUMP' command."); } parserConfigs.put(sensorType, parserConfig); } } return parserConfigs; }
private void waitForIndex(String zookeeperQuorum) throws Exception { try(CuratorFramework client = getClient(zookeeperQuorum)) { client.start(); System.out.println("Waiting for zookeeper..."); byte[] bytes = null; do { try { bytes = ConfigurationsUtils.readSensorIndexingConfigBytesFromZookeeper(testSensorType, client); Thread.sleep(1000); } catch(KeeperException.NoNodeException nne) { //kindly ignore because the path might not exist just yet. } } while(bytes == null || bytes.length == 0); System.out.println("Found index config in zookeeper..."); } }
public SensorParserConfig getSensorParserConfig(String sensorType) { SensorParserConfig sensorParserConfig = new SensorParserConfig(); CuratorFramework client = getClient(topologyProperties.getProperty(ZKServerComponent.ZOOKEEPER_PROPERTY)); client.start(); try { sensorParserConfig = readSensorParserConfigFromZookeeper(sensorType, client); } catch (Exception e) { e.printStackTrace(); } finally { client.close(); } return sensorParserConfig; }
/** * Uploads configuration to Zookeeper. * @throws Exception */ private void upload() throws Exception { assert zookeeperURL != null; try(CuratorFramework client = getClient(zookeeperURL)) { if(client.getState() != CuratorFrameworkState.STARTED) { client.start(); } if (globals != null) { writeGlobalConfigToZookeeper(globals, client); } } }
@Before public void setup() throws Exception { testZkServer = new TestingServer(true); zookeeperUrl = testZkServer.getConnectString(); client = ConfigurationsUtils.getClient(zookeeperUrl); client.start(); File sensorDir = new File(new File(TestConstants.SAMPLE_CONFIG_PATH), ENRICHMENT.getDirectory()); sensors.addAll(Collections2.transform( Arrays.asList(sensorDir.list()) , s -> Iterables.getFirst(Splitter.on('.').split(s), "null"))); tmpDir = TestUtils.createTempDir(this.getClass().getName()); configDir = TestUtils.createDir(tmpDir, "config"); parsersDir = TestUtils.createDir(configDir, "parsers"); enrichmentsDir = TestUtils.createDir(configDir, "enrichments"); indexingDir = TestUtils.createDir(configDir, "indexing"); pushAllConfigs(); }
zkComponent = new ZKServerComponent(); zkComponent.start(); client = ConfigurationsUtils.getClient(zkComponent.getConnectionString()); client.start(); cache = new ZKConfigurationsCache(client);
@Before public void setup() throws Exception { testZkServer = new TestingServer(true); zookeeperUrl = testZkServer.getConnectString(); client = ConfigurationsUtils.getClient(zookeeperUrl); client.start(); expectedGlobalConfig = ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH); expectedSensorParserConfigMap = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH); expectedSensorEnrichmentConfigMap = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH); }