@Override public String getIndex(String sensorName) { if(config != null && config.getSensorParserConfig(sensorName) != null && config.getSensorParserConfig(sensorName).getParserConfig() != null ) { Object indexObj = config.getSensorParserConfig(sensorName).getParserConfig().get(IndexingConfigurations.INDEX_CONF); if(indexObj != null) { return indexObj.toString(); } return null; } return sensorName; }
public StellarParserRunner withParserConfiguration(String sensorType, SensorParserConfig config) { parserConfigurations = new ParserConfigurations(); parserConfigurations.updateSensorParserConfig(sensorType, config); return this; }
private List<FieldValidator> getFailedValidators(JSONObject message, ParserConfigurations parserConfigurations) { List<FieldValidator> fieldValidations = parserConfigurations.getFieldValidations(); List<FieldValidator> failedValidators = new ArrayList<>(); for(FieldValidator validator : fieldValidations) { if(!validator.isValid(message, parserConfigurations.getGlobalConfig(), stellarContext)) { failedValidators.add(validator); } } return failedValidators; } }
public SensorParserConfig getSensorParserConfig(String sensorType) { return (SensorParserConfig) getConfigurations().get(getKey(sensorType)); }
public static ParserConfigurations getSampleParserConfigs() throws IOException { ParserConfigurations configurations = new ParserConfigurations(); configurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH)); Map<String, byte[]> sensorParserConfigs = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH); for(String sensorType: sensorParserConfigs.keySet()) { configurations.updateSensorParserConfig(sensorType, sensorParserConfigs.get(sensorType)); } return configurations; }
@Test public void test() throws Exception { ParserConfigurations sampleConfigurations = new ParserConfigurations(); UnitTestHelper.setLog4jLevel(ConfiguredBolt.class, Level.FATAL); try { sampleConfigurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH)); Map<String, byte[]> sensorParserConfigs = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH); for (String sensorType : sensorParserConfigs.keySet()) { sampleConfigurations.updateSensorParserConfig(sensorType, sensorParserConfigs.get(sensorType)); Map<String, Object> sampleGlobalConfig = sampleConfigurations.getGlobalConfig(); sampleGlobalConfig.put("newGlobalField", "newGlobalValue"); ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl); waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName()); Assert.assertEquals("Add global config field", sampleConfigurations.getGlobalConfig(), configuredBolt.getConfigurations().getGlobalConfig()); put("configName", "configObject"); }}); sampleConfigurations.updateSensorParserConfig(sensorType, testSensorConfig); ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorType, testSensorConfig, zookeeperUrl); waitForConfigUpdate(sensorType); ParserConfigurations configuredBoltConfigs = configuredBolt.getConfigurations(); if(!sampleConfigurations.equals(configuredBoltConfigs)) { if(sampleConfigurations.getFieldValidations().size() != configuredBoltConfigs.getFieldValidations().size()) { System.out.println("Field validations don't line up"); for(int i = 0;i < sampleConfigurations.getFieldValidations().size();++i) { FieldValidator l = sampleConfigurations.getFieldValidations().get(i);
@Test public void sensorParserConfig_properties_populated_by_JSON_configuration() throws IOException { ParserConfigurations parserConfigs = new ParserConfigurations(); parserConfigs.updateSensorParserConfig("test-sensor", parserConfig.getBytes()); SensorParserConfig actualSensorConfig = parserConfigs.getSensorParserConfig("test-sensor"); assertThat(actualSensorConfig.getParserClassName(), equalTo("parser-class")); assertThat(actualSensorConfig.getFilterClassName(), equalTo("filter-class"));
SensorParserConfig expectedConfig = JSONUtils.INSTANCE.load(inFile, SensorParserConfig.class); ParserConfigurations config = cache.get( ParserConfigurations.class); assertEventually(() -> Assert.assertEquals(expectedConfig, config.getSensorParserConfig("bro"))); assertEventually(() -> Assert.assertEquals(expectedGlobalConfig, config.getGlobalConfig())); assertEventually(() -> Assert.assertNull(config.getSensorParserConfig("notthere")));
@Override public ParserConfigurations defaultConfigurations() { return new ParserConfigurations(); }
public void updateSensorParserConfig(String sensorType, byte[] data) throws IOException { updateSensorParserConfig(sensorType, new ByteArrayInputStream(data)); }
@Override public Map<String, Object> getGlobalConfig() { return config.getGlobalConfig(); }
public StellarParserRunner withGlobals(Map<String, Object> globals) { parserConfigurations.updateGlobalConfig(globals); return this; }
public List<String> getTypes() { List<String> ret = new ArrayList<>(); for(String keyedSensor : getConfigurations().keySet()) { if(!keyedSensor.isEmpty() && keyedSensor.startsWith(ConfigurationType.PARSER.getTypeName())) { ret.add(keyedSensor.substring(ConfigurationType.PARSER.getTypeName().length() + 1)); } } return ret; }
public ParserDriver(String sensorType, String parserConfig, String globalConfig) throws IOException { SensorParserConfig sensorParserConfig = SensorParserConfig.fromBytes(parserConfig.getBytes()); this.sensorType = sensorType == null ? sensorParserConfig.getSensorTopic() : sensorType; config = new ParserConfigurations(); config.updateSensorParserConfig(this.sensorType, SensorParserConfig.fromBytes(parserConfig.getBytes())); config.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER)); parserRunner = new ParserRunnerImpl(new HashSet<String>() {{ add(sensorType); }}); }
assertEventually(() -> Assert.assertNull(config.getSensorParserConfig("bro"))); assertEventually(() -> Assert.assertNull(config.getGlobalConfig(false)));
@Test public void testDefaultIndex() { ParserWriterConfiguration config = new ParserWriterConfiguration( new ParserConfigurations() ); Assert.assertEquals("foo", config.getIndex("foo")); }
public void updateSensorParserConfig(String sensorType, InputStream io) throws IOException { SensorParserConfig sensorParserConfig = JSONUtils.INSTANCE.load(io, SensorParserConfig.class); updateSensorParserConfig(sensorType, sensorParserConfig); }
protected Context initializeStellar() { Map<String, Object> cacheConfig = new HashMap<>(); for (String sensorType: this.parserRunner.getSensorTypes()) { SensorParserConfig config = getSensorParserConfig(sensorType); if (config != null) { cacheConfig.putAll(config.getCacheConfig()); } } Cache<CachingStellarProcessor.Key, Object> cache = CachingStellarProcessor.createCache(cacheConfig); Context.Builder builder = new Context.Builder() .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client) .with(Context.Capabilities.GLOBAL_CONFIG, () -> getConfigurations().getGlobalConfig()) .with(Context.Capabilities.STELLAR_CONFIG, () -> getConfigurations().getGlobalConfig()) ; if(cache != null) { builder = builder.with(Context.Capabilities.CACHE, () -> cache); } Context stellarContext = builder.build(); StellarFunctions.initialize(stellarContext); return stellarContext; }
public void delete(String sensorType) { getConfigurations().remove(getKey(sensorType)); }
private ParserConfigurations create(byte[] sensorConfig) { try { ParserConfigurations result = new ParserConfigurations(); result.updateSensorParserConfig(sensorType, SensorParserConfig.fromBytes(sensorConfig)); return result; } catch(IOException e) { throw new IllegalArgumentException(e); } }