public static void writeGlobalConfigToZookeeper(byte[] globalConfig, String zookeeperUrl) throws Exception { try(CuratorFramework client = getClient(zookeeperUrl)) { client.start(); writeGlobalConfigToZookeeper(globalConfig, client); } }
final byte[] globalConfig = readGlobalConfigFromFile(globalConfigPath); if (globalConfig.length > 0) { setupStellarStatically(client, Optional.of(new String(globalConfig))); ConfigurationsUtils.writeGlobalConfigToZookeeper(readGlobalConfigFromFile(globalConfigPath), client); Map<String, byte[]> sensorParserConfigs = readSensorParserConfigsFromFile(parsersConfigPath); for (String sensorType : sensorParserConfigs.keySet()) { ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorType, sensorParserConfigs.get(sensorType), client); Map<String, byte[]> sensorIndexingConfigs = readSensorIndexingConfigsFromFile(indexingConfigPath); for (String sensorType : sensorIndexingConfigs.keySet()) { ConfigurationsUtils.writeSensorIndexingConfigToZookeeper(sensorType, sensorIndexingConfigs.get(sensorType), client); Map<String, byte[]> sensorEnrichmentConfigs = readSensorEnrichmentConfigsFromFile(enrichmentsConfigPath); for (String sensorType : sensorEnrichmentConfigs.keySet()) { ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(sensorType, sensorEnrichmentConfigs.get(sensorType), client); byte[] profilerConfig = readProfilerConfigFromFile(profilerConfigPath); if (profilerConfig.length > 0) { ConfigurationsUtils.writeProfilerConfigToZookeeper(profilerConfig, client);
protected void pushConfig(Path srcPath, Path dstPath, String configName, String zookeeper) { System.out.println("Beginning update of global configs"); try (CuratorFramework client = ConfigurationsUtils.getClient(zookeeper)) { client.start(); // Use the parent and place a new file. Has to be a new file so we can update the configs and trigger updates. // Fetch the global configuration Map<String, Object> global = JSONUtils.INSTANCE.load( new ByteArrayInputStream(ConfigurationsUtils.readGlobalConfigBytesFromZookeeper(client)), JSONUtils.MAP_SUPPLIER); // Update the global config and push it back global.put(configName, dstPath.toString() + "/" + srcPath.getName()); ConfigurationsUtils.writeGlobalConfigToZookeeper(global, client); } catch (Exception e) { System.err.println("Unable to load new GeoLite2 config for " + configName + " into HDFS: " + e); e.printStackTrace(); System.exit(2); } System.out.println("Finished update of global configs"); }
public static void writeSensorEnrichmentConfigToZookeeper(String sensorType, byte[] configData, String zookeeperUrl) throws Exception { try(CuratorFramework client = getClient(zookeeperUrl)) { client.start(); writeSensorEnrichmentConfigToZookeeper(sensorType, configData, client); } }
public static void uploadConfigsToZookeeper(String globalConfigPath, String parsersConfigPath, String enrichmentsConfigPath, String indexingConfigPath, String profilerConfigPath, String zookeeperUrl) throws Exception { try (CuratorFramework client = getClient(zookeeperUrl)) { client.start(); uploadConfigsToZookeeper(globalConfigPath, parsersConfigPath, enrichmentsConfigPath, indexingConfigPath, profilerConfigPath, client); } }
writeGlobalConfigToZookeeper(value.getBytes(), client); writeProfilerConfigToZookeeper(value.getBytes(), client); writeSensorEnrichmentConfigToZookeeper(sensor, value.getBytes(), client); writeSensorIndexingConfigToZookeeper(sensor, value.getBytes(), client); writeSensorParserConfigToZookeeper(sensor, value.getBytes(), client);
@Before public void setupConfiguration() throws Exception { TestingServer testZkServer = new TestingServer(true); this.zookeeperUrl = testZkServer.getConnectString(); byte[] globalConfig = ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH); ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig, zookeeperUrl); enrichmentConfigurationTypes.add(ConfigurationType.GLOBAL.getTypeName()); Map<String, byte[]> sensorEnrichmentConfigs = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH); for (String sensorType : sensorEnrichmentConfigs.keySet()) { ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(sensorType, sensorEnrichmentConfigs.get(sensorType), zookeeperUrl); enrichmentConfigurationTypes.add(sensorType); } Map<String, byte[]> sensorParserConfigs = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH); for (String sensorType : sensorParserConfigs.keySet()) { ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorType, sensorParserConfigs.get(sensorType), zookeeperUrl); } }
@Test public void test() throws Exception { Assert.assertTrue(expectedGlobalConfig.length > 0); ConfigurationsUtils.writeGlobalConfigToZookeeper(expectedGlobalConfig, zookeeperUrl); byte[] actualGlobalConfigBytes = ConfigurationsUtils.readGlobalConfigBytesFromZookeeper(client); Assert.assertTrue(Arrays.equals(expectedGlobalConfig, actualGlobalConfigBytes)); Assert.assertTrue(expectedSensorParserConfigMap.size() > 0); String testSensorType = "yaf"; byte[] expectedSensorParserConfigBytes = expectedSensorParserConfigMap.get(testSensorType); ConfigurationsUtils.writeSensorParserConfigToZookeeper(testSensorType, expectedSensorParserConfigBytes, zookeeperUrl); byte[] actualSensorParserConfigBytes = ConfigurationsUtils.readSensorParserConfigBytesFromZookeeper(testSensorType, client); Assert.assertTrue(Arrays.equals(expectedSensorParserConfigBytes, actualSensorParserConfigBytes)); Assert.assertTrue(expectedSensorEnrichmentConfigMap.size() > 0); byte[] expectedSensorEnrichmentConfigBytes = expectedSensorEnrichmentConfigMap.get(testSensorType); ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(testSensorType, expectedSensorEnrichmentConfigBytes, zookeeperUrl); byte[] actualSensorEnrichmentConfigBytes = ConfigurationsUtils.readSensorEnrichmentConfigBytesFromZookeeper(testSensorType, client); Assert.assertTrue(Arrays.equals(expectedSensorEnrichmentConfigBytes, actualSensorEnrichmentConfigBytes)); String name = "testConfig"; Map<String, Object> testConfig = new HashMap<>(); testConfig.put("stringField", "value"); testConfig.put("intField", 1); testConfig.put("doubleField", 1.1); ConfigurationsUtils.writeConfigToZookeeper(name, testConfig, zookeeperUrl); byte[] readConfigBytes = ConfigurationsUtils.readConfigBytesFromZookeeper(name, client); Assert.assertTrue(Arrays.equals(JSONUtils.INSTANCE.toJSONPretty(testConfig), readConfigBytes)); }
final byte[] globalConfig = readGlobalConfigFromFile(rootFilePath); if (globalConfig.length > 0) { setupStellarStatically(client, Optional.of(new String(globalConfig))); writeGlobalConfigToZookeeper(globalConfig, client); case INDEXING: //pass through intentional Map<String, byte[]> configs = readSensorConfigsFromFile(rootFilePath, type, configName); for (String sensorType : configs.keySet()) { byte[] configData = configs.get(sensorType); byte[] configData = readProfilerConfigFromFile(rootFilePath); if (configData.length > 0) { ConfigurationsUtils.writeProfilerConfigToZookeeper(configData, client);
sampleConfigurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH)); Map<String, byte[]> sensorParserConfigs = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH); for (String sensorType : sensorParserConfigs.keySet()) { sampleConfigurations.updateSensorParserConfig(sensorType, sensorParserConfigs.get(sensorType)); Map<String, Object> sampleGlobalConfig = sampleConfigurations.getGlobalConfig(); sampleGlobalConfig.put("newGlobalField", "newGlobalValue"); ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl); waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName()); Assert.assertEquals("Add global config field", sampleConfigurations.getGlobalConfig(), configuredBolt.getConfigurations().getGlobalConfig()); ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl); waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName()); Assert.assertEquals("Remove global config field", sampleConfigurations, configuredBolt.getConfigurations()); }}); sampleConfigurations.updateSensorParserConfig(sensorType, testSensorConfig); ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorType, testSensorConfig, zookeeperUrl); waitForConfigUpdate(sensorType); ParserConfigurations configuredBoltConfigs = configuredBolt.getConfigurations();
@Before public void setup() throws Exception { testZkServer = new TestingServer(true); zookeeperUrl = testZkServer.getConnectString(); client = ConfigurationsUtils.getClient(zookeeperUrl); client.start(); expectedGlobalConfig = ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH); expectedSensorParserConfigMap = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH); expectedSensorEnrichmentConfigMap = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH); }
sampleConfigurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH)); Map<String, byte[]> sensorEnrichmentConfigs = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH); for (String sensorType : sensorEnrichmentConfigs.keySet()) { sampleConfigurations.updateSensorEnrichmentConfig(sensorType, sensorEnrichmentConfigs.get(sensorType)); Map<String, Object> sampleGlobalConfig = sampleConfigurations.getGlobalConfig(); sampleGlobalConfig.put("newGlobalField", "newGlobalValue"); ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl); waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName()); Assert.assertEquals("Add global config field", sampleConfigurations.getGlobalConfig(), configuredBolt.getConfigurations().getGlobalConfig()); ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl); waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName()); Assert.assertEquals("Remove global config field", sampleConfigurations, configuredBolt.getConfigurations()); testSensorConfig.getThreatIntel().setFieldMap(threatIntelFieldMap); sampleConfigurations.updateSensorEnrichmentConfig(sensorType, testSensorConfig); ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(sensorType, testSensorConfig, zookeeperUrl); waitForConfigUpdate(sensorType); Assert.assertEquals("Add new sensor config", sampleConfigurations, configuredBolt.getConfigurations());
public static void writeSensorParserConfigToZookeeper(String sensorType, byte[] configData, String zookeeperUrl) throws Exception { try(CuratorFramework client = getClient(zookeeperUrl)) { client.start(); writeSensorParserConfigToZookeeper(sensorType, configData, client); } }
public void update() throws UnableToStartException { try { final String zookeeperUrl = connectionString == null?topologyProperties.getProperty(ZKServerComponent.ZOOKEEPER_PROPERTY):connectionString; if(globalConfigPath != null || parserConfigsPath != null || enrichmentConfigsPath != null || indexingConfigsPath != null || profilerConfigPath != null ) { uploadConfigsToZookeeper(globalConfigPath, parserConfigsPath, enrichmentConfigsPath, indexingConfigsPath, profilerConfigPath, zookeeperUrl); } for(Map.Entry<String, SensorParserConfig> kv : parserSensorConfigs.entrySet()) { writeSensorParserConfigToZookeeper(kv.getKey(), kv.getValue(), zookeeperUrl); } if(globalConfig.isPresent()) { writeGlobalConfigToZookeeper(globalConfig.get().getBytes(), zookeeperUrl); } if(postStartCallback.isPresent()) { postStartCallback.get().accept(this); } } catch (Exception e) { throw new UnableToStartException(e.getMessage(), e); } }
public static void writeSensorIndexingConfigToZookeeper(String sensorType, byte[] configData, String zookeeperUrl) throws Exception { try(CuratorFramework client = getClient(zookeeperUrl)) { client.start(); writeSensorIndexingConfigToZookeeper(sensorType, configData, client); } }
/** * Creates a Zookeeper client if it doesn't exist and a url for zk is provided. * @param zookeeperUrl The Zookeeper URL. */ private Optional<CuratorFramework> setupClient(Optional<CuratorFramework> zkClient, String zookeeperUrl) { // can only create client if we have a valid zookeeper URL if (!zkClient.isPresent()) { if (StringUtils.isNotBlank(zookeeperUrl)) { CuratorFramework client = ConfigurationsUtils.getClient(zookeeperUrl); client.start(); return Optional.of(client); } else { LOG.warn("Unable to setup zookeeper client - zk_quorum url not provided. **This will limit some Stellar functionality**"); return Optional.empty(); } } else { return zkClient; } }
/** * Reads Json data for the specified config type and config name (if applicable) from zookeeper, * applies the patch from patchData, and writes it back to Zookeeper in a pretty print format. * Patching JSON flattens existing formatting, so this will keep configs readable. * Starts up curatorclient based on zookeeperUrl. * * @param configurationType GLOBAL, PARSER, etc. * @param configName e.g. bro, yaf, snort * @param patchData a JSON patch in the format specified by RFC 6902 * @param zookeeperUrl configs are here */ public static void applyConfigPatchToZookeeper(ConfigurationType configurationType, Optional<String> configName, byte[] patchData, String zookeeperUrl) throws Exception { try (CuratorFramework client = getClient(zookeeperUrl)) { client.start(); applyConfigPatchToZookeeper(configurationType, configName, patchData, client); } }
public static byte[] readFromZookeeper(String path, String zookeeperUrl) throws Exception { try (CuratorFramework client = getClient(zookeeperUrl)) { client.start(); return readFromZookeeper(path, client); } }
/** * Writes config to path in Zookeeper, /metron/topology/$CONFIG_TYPE/$CONFIG_NAME */ public static void writeConfigToZookeeper(String configPath, byte[] config, String zookeeperUrl) throws Exception { try (CuratorFramework client = getClient(zookeeperUrl)) { client.start(); writeToZookeeper(configPath, config, client); } }
/** * Fetch the parser configuration from Zookeeper. * * @param zookeeperUrl Zookeeper URL * @param sensorTypes Types of sensor * @param configs * @return * @throws Exception */ private static Map<String, SensorParserConfig> getSensorParserConfig(String zookeeperUrl, List<String> sensorTypes, ParserConfigurations configs) throws Exception { Map<String, SensorParserConfig> parserConfigs = new HashMap<>(); try(CuratorFramework client = ConfigurationsUtils.getClient(zookeeperUrl)) { client.start(); ConfigurationsUtils.updateParserConfigsFromZookeeper(configs, client); for (String sensorType : sensorTypes) { SensorParserConfig parserConfig = configs.getSensorParserConfig(sensorType); if (parserConfig == null) { throw new IllegalStateException("Cannot find the parser configuration in zookeeper for " + sensorType + "." + " Please check that it exists in zookeeper by using the 'zk_load_configs.sh -m DUMP' command."); } parserConfigs.put(sensorType, parserConfig); } } return parserConfigs; }