@Override public Map<String, Object> get() throws RestException { Map<String, Object> globalConfig; try { EnrichmentConfigurations configs = cache.get( EnrichmentConfigurations.class); globalConfig = configs.getGlobalConfig(false); } catch (Exception e) { throw new RestException(e.getMessage(), e); } return globalConfig; }
@Override public SensorEnrichmentConfig findOne(String name) throws RestException { EnrichmentConfigurations configs = cache.get( EnrichmentConfigurations.class); return configs.getSensorEnrichmentConfig(name); }
@Override public Map<String, Object> getGlobalConfig() { return config.orElse(new EnrichmentConfigurations()).getGlobalConfig(); }
/** * Pulled from global config * Note: enrichment writes out to 1 kafka topic, so it is not pulling this config by sensor. * * @return batch timeout for writing to kafka * @see org.apache.metron.common.configuration.EnrichmentConfigurations#BATCH_TIMEOUT_CONF */ public int getBatchTimeout() { return getAs(BATCH_TIMEOUT_CONF, getGlobalConfig(true), 0, Integer.class); }
/** * Timeout for this writer. * @param sensorName n/a * @return timeout in ms */ @Override public int getBatchTimeout(String sensorName) { return config.orElse(new EnrichmentConfigurations()).getBatchTimeout(); }
/** * Batch size for writing. * @param sensorName n/a * @return batch size in # messages */ @Override public int getBatchSize(String sensorName) { return config.orElse(new EnrichmentConfigurations()).getBatchSize(); }
Map<String, Object> expectedGlobalConfig = JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER); EnrichmentConfigurations config = cache.get( EnrichmentConfigurations.class); assertEventually(() -> Assert.assertEquals(expectedConfig, config.getSensorEnrichmentConfig("test"))); assertEventually(() -> Assert.assertEquals(expectedGlobalConfig, config.getGlobalConfig()));
@Test public void test() throws Exception { EnrichmentConfigurations sampleConfigurations = new EnrichmentConfigurations(); UnitTestHelper.setLog4jLevel(ConfiguredBolt.class, Level.FATAL); try { sampleConfigurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH)); Map<String, byte[]> sensorEnrichmentConfigs = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH); for (String sensorType : sensorEnrichmentConfigs.keySet()) { sampleConfigurations.updateSensorEnrichmentConfig(sensorType, sensorEnrichmentConfigs.get(sensorType)); Map<String, Object> sampleGlobalConfig = sampleConfigurations.getGlobalConfig(); sampleGlobalConfig.put("newGlobalField", "newGlobalValue"); ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl); waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName()); Assert.assertEquals("Add global config field", sampleConfigurations.getGlobalConfig(), configuredBolt.getConfigurations().getGlobalConfig()); }}); testSensorConfig.getThreatIntel().setFieldMap(threatIntelFieldMap); sampleConfigurations.updateSensorEnrichmentConfig(sensorType, testSensorConfig); ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(sensorType, testSensorConfig, zookeeperUrl); waitForConfigUpdate(sensorType);
public static EnrichmentConfigurations getSampleEnrichmentConfigs() throws IOException { EnrichmentConfigurations configurations = new EnrichmentConfigurations(); configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH)); Map<String, byte[]> sensorEnrichmentConfigs = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.SAMPLE_CONFIG_PATH); for(String sensorType: sensorEnrichmentConfigs.keySet()) { configurations.updateSensorEnrichmentConfig(sensorType, sensorEnrichmentConfigs.get(sensorType)); } return configurations; }
@Override public Map<String, Object> getSensorConfig(String sensorName) { return config.orElse(new EnrichmentConfigurations()).getSensorEnrichmentConfig(sensorName) .getConfiguration(); }
public void updateSensorEnrichmentConfig(String sensorType, byte[] data) throws IOException { updateSensorEnrichmentConfig(sensorType, new ByteArrayInputStream(data)); }
threatIntelJoinBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, enrichmentConfig); HashMap<String, Object> globalConfig = new HashMap<>(); String baseDir = UnitTestHelper.findDir("GeoLite"); File asnHdfsFile = new File(new File(baseDir), "GeoLite2-ASN.tar.gz"); globalConfig.put(GeoLiteAsnDatabase.ASN_HDFS_FILE, asnHdfsFile.getAbsolutePath()); threatIntelJoinBolt.getConfigurations().updateGlobalConfig(globalConfig); threatIntelJoinBolt.withMaxCacheSize(100); threatIntelJoinBolt.withMaxTimeRetain(10000);
@Test public void gets_batch_size_and_timeout_from_global_config() throws IOException { EnrichmentConfigurations configs = new EnrichmentConfigurations(); configs.updateGlobalConfig(globalJson.getBytes()); EnrichmentWriterConfiguration writerConfig = new EnrichmentWriterConfiguration(configs); assertThat("batch timeout should match global config setting", writerConfig.getBatchTimeout(null), equalTo(555)); assertThat("list should have single batch timeout matching global config setting", writerConfig.getAllConfiguredTimeouts(), equalTo(asList(555))); assertThat("batch size should match global config setting", writerConfig.getBatchSize(null), equalTo(12345)); }
@Override public EnrichmentConfigurations defaultConfigurations() { return new EnrichmentConfigurations(); }
@Override public void delete(String name) { getConfigurations().delete(name); }
SensorEnrichmentConfig expectedConfig = JSONUtils.INSTANCE.load(inFile, SensorEnrichmentConfig.class); EnrichmentConfigurations config = cache.get( EnrichmentConfigurations.class); assertEventually(() -> Assert.assertEquals(expectedConfig, config.getSensorEnrichmentConfig("test"))); assertEventually(() -> Assert.assertEquals(expectedGlobalConfig, config.getGlobalConfig())); assertEventually(() -> Assert.assertNull(config.getSensorEnrichmentConfig("notthere")));
@Override public void update(String name, byte[] data) throws IOException { getConfigurations().updateSensorEnrichmentConfig(name, data); }
/** * Pulled from global config. * Note: enrichment writes out to 1 kafka topic, so it is not pulling this config by sensor. * * @return batch size for writing to kafka * @see org.apache.metron.common.configuration.EnrichmentConfigurations#BATCH_SIZE_CONF */ public int getBatchSize() { return getAs(BATCH_SIZE_CONF, getGlobalConfig(true), DEFAULT_KAFKA_BATCH_SIZE, Integer.class); }
genericEnrichmentBolt.setCuratorFramework(client); genericEnrichmentBolt.setZKCache(cache); genericEnrichmentBolt.getConfigurations().updateSensorEnrichmentConfig(sensorType, new FileInputStream(sampleSensorEnrichmentConfigPath)); File geoHdfsFile = new File(new File(baseDir), "GeoLite2-City.mmdb.gz"); globalConfig.put(GeoLiteCityDatabase.GEO_HDFS_FILE, geoHdfsFile.getAbsolutePath()); genericEnrichmentBolt.getConfigurations().updateGlobalConfig(globalConfig);
@Test public void strategies_build_writer_configs() { assertThat(PARSERS.createWriterConfig(writer, new ParserConfigurations()), instanceOf(ParserWriterConfiguration.class)); assertThat(ENRICHMENT.createWriterConfig(writer, new EnrichmentConfigurations()), instanceOf(EnrichmentWriterConfiguration.class)); assertThat(INDEXING.createWriterConfig(writer, new IndexingConfigurations()), instanceOf(IndexingWriterConfiguration.class)); assertThat(PROFILER.createWriterConfig(writer, new ProfilerConfigurations()), instanceOf(ProfilerWriterConfiguration.class)); }