- Add the Codota plugin to your IDE and get smart completions
private void myMethod () {LocalDateTime l =
new LocalDateTime()
- Smart code suggestions by Tabnine
}
private List<JSONObject> doParse(List<String> messages) { // initialize HashSet<String> sensorTypes = new HashSet<>(); sensorTypes.add(sensorType); ParserRunnerImpl runner = new ParserRunnerImpl(sensorTypes); runner.init(() -> parserConfigurations, context); // parse each message List<ParserRunnerResults<JSONObject>> results = messages .stream() .map(str -> str.getBytes()) .map(bytes -> DEFAULT.get(emptyMap(), bytes, false, emptyMap())) .map(msg -> runner.execute(sensorType, msg, parserConfigurations)) .collect(Collectors.toList()); // aggregate both successes and errors into a list that can be returned List<JSONObject> successes = results .stream() .flatMap(result -> result.getMessages().stream()) .collect(Collectors.toList()); successCount += successes.size(); List<JSONObject> errors = results .stream() .flatMap(result -> result.getErrors().stream()) .map(err -> err.getJSONObject()) .collect(Collectors.toList()); errorCount += errors.size(); // return a list of both successes and errors successes.addAll(errors); return successes; }
@Override public void init(Supplier<ParserConfigurations> parserConfigSupplier, Context stellarContext) { if (parserConfigSupplier == null) { throw new IllegalStateException("A parser config supplier must be set before initializing the ParserRunner."); } if (stellarContext == null) { throw new IllegalStateException("A stellar context must be set before initializing the ParserRunner."); } this.stellarContext = stellarContext; initializeParsers(parserConfigSupplier); }
@Test public void initShouldThrowExceptionOnMissingSensorParserConfig() { exception.expect(IllegalStateException.class); exception.expectMessage("Could not initialize parsers. Cannot find configuration for sensor test."); parserRunner = new ParserRunnerImpl(new HashSet<String>() {{ add("test"); }}); parserRunner.init(() -> parserConfigurations, mock(Context.class)); }
@Test public void executeShouldThrowExceptionOnMissingSensorParserConfig() { exception.expect(IllegalStateException.class); exception.expectMessage("Could not execute parser. Cannot find configuration for sensor test."); parserRunner = new ParserRunnerImpl(new HashSet<String>() {{ add("test"); }}); parserRunner.execute("test", mock(RawMessage.class), parserConfigurations); }
@Test public void shouldExecuteWithMasterThrowable() { parserRunner = spy(parserRunner); RawMessage rawMessage = new RawMessage("raw_message".getBytes(), new HashMap<>()); Throwable masterThrowable = mock(Throwable.class); MessageParserResult<JSONObject> messageParserResult = new DefaultMessageParserResult<>(masterThrowable); when(broParser.parseOptionalResult(rawMessage.getMessage())).thenReturn(Optional.of(messageParserResult)); parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {{ put("bro", new ParserComponent(broParser, stellarFilter)); }}); ParserRunnerResults<JSONObject> parserRunnerResults = parserRunner.execute("bro", rawMessage, parserConfigurations); verify(parserRunner, times(0)) .processMessage(any(), any(), any(), any(), any()); MetronError expectedError = new MetronError() .withErrorType(Constants.ErrorType.PARSER_ERROR) .withThrowable(masterThrowable) .withSensorType(Collections.singleton("bro")) .addRawMessage(rawMessage.getMessage()); Assert.assertEquals(1, parserRunnerResults.getErrors().size()); Assert.assertTrue(parserRunnerResults.getErrors().contains(expectedError)); }
@Test public void shouldPopulateMessagesOnProcessMessage() { JSONObject inputMessage = new JSONObject(); inputMessage.put("guid", "guid"); inputMessage.put("ip_src_addr", "192.168.1.1"); inputMessage.put("ip_dst_addr", "192.168.1.2"); inputMessage.put("field1", "value"); RawMessage rawMessage = new RawMessage("raw_message".getBytes(), new HashMap<>()); JSONObject expectedOutput = new JSONObject(); expectedOutput.put("guid", "guid"); expectedOutput.put("source.type", "bro"); expectedOutput.put("ip_src_addr", "192.168.1.1"); expectedOutput.put("ip_dst_addr", "192.168.1.2"); when(stellarFilter.emit(expectedOutput, parserRunner.getStellarContext())).thenReturn(true); when(broParser.validate(expectedOutput)).thenReturn(true); parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {{ put("bro", new ParserComponent(broParser, stellarFilter)); }}); Optional<ParserRunnerImpl.ProcessResult> processResult = parserRunner.processMessage("bro", inputMessage, rawMessage, broParser, parserConfigurations); Assert.assertTrue(processResult.isPresent()); Assert.assertFalse(processResult.get().isError()); Assert.assertEquals(expectedOutput, processResult.get().getMessage()); }
Map<String, Object> snortParserConfig = parserConfigurations.getSensorParserConfig("snort").getParserConfig(); parserRunner.init(() -> parserConfigurations, stellarContext); Assert.assertEquals(stellarContext, parserRunner.getStellarContext()); Map<String, ParserComponent> sensorToParserComponentMap = parserRunner.getSensorToParserComponentMap();
@Test public void shouldThrowExceptionOnEmptyParserSupplier() { exception.expect(IllegalStateException.class); exception.expectMessage("A parser config supplier must be set before initializing the ParserRunner."); parserRunner.init(null, null); }
return new ParserBolt(zookeeperUrl, new ParserRunnerImpl(new HashSet<>(sensorTypeToParserConfig.keySet())), writerConfigs);
doThrow(new IllegalStateException("parserRunner.execute failed")).when(parserRunner).execute(eq("yaf"), any(), eq(parserConfigurations));
); message.put(Constants.SENSOR_TYPE, sensorType); applyFieldTransformations(message, rawMessage, sensorParserConfig); if (!message.containsKey(Constants.GUID)) { message.put(Constants.GUID, UUID.randomUUID().toString()); List<FieldValidator> failedValidators = null; if (!isInvalid) { failedValidators = getFailedValidators(message, parserConfigurations); isInvalid = !failedValidators.isEmpty();
Optional<ProcessResult> processResult = processMessage(sensorType, message, rawMessage, parser, parserConfigurations); if (processResult.isPresent()) { if (processResult.get().isError()) {
when(processedErrorResult.getError()).thenReturn(processedError); doReturn(Optional.of(processedMessageResult)).when(parserRunner) .processMessage("bro", parsedMessage1, rawMessage, broParser, parserConfigurations); doReturn(Optional.of(processedErrorResult)).when(parserRunner) .processMessage("bro", parsedMessage2, rawMessage, broParser, parserConfigurations); .addRawMessage(rawMessage2); parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {{ put("bro", new ParserComponent(broParser, stellarFilter)); }}); ParserRunnerResults<JSONObject> parserRunnerResults = parserRunner.execute("bro", rawMessage, parserConfigurations);
@Test public void shouldReturnMetronErrorOnInvalidMessage() { JSONObject inputMessage = new JSONObject(); inputMessage.put("guid", "guid"); RawMessage rawMessage = new RawMessage("raw_message".getBytes(), new HashMap<>()); JSONObject expectedOutput = new JSONObject(); expectedOutput.put("guid", "guid"); expectedOutput.put("source.type", "bro"); MetronError expectedMetronError = new MetronError() .withErrorType(Constants.ErrorType.PARSER_INVALID) .withSensorType(Collections.singleton("bro")) .addRawMessage(inputMessage); when(stellarFilter.emit(expectedOutput, parserRunner.getStellarContext())).thenReturn(true); when(broParser.validate(expectedOutput)).thenReturn(false); parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {{ put("bro", new ParserComponent(broParser, stellarFilter)); }}); Optional<ParserRunnerImpl.ProcessResult> processResult = parserRunner.processMessage("bro", inputMessage, rawMessage, broParser, parserConfigurations); Assert.assertTrue(processResult.isPresent()); Assert.assertTrue(processResult.get().isError()); Assert.assertEquals(expectedMetronError, processResult.get().getError()); }
@Test public void shouldThrowExceptionOnEmptyStellarContext() { exception.expect(IllegalStateException.class); exception.expectMessage("A stellar context must be set before initializing the ParserRunner."); parserRunner.init(() -> parserConfigurations, null); }
@Before public void setup() throws IOException { parserConfigurations = new ParserConfigurations(); SensorParserConfig broConfig = SensorParserConfig.fromBytes(broConfigString.getBytes()); SensorParserConfig snortConfig = SensorParserConfig.fromBytes(snortConfigString.getBytes()); parserConfigurations.updateSensorParserConfig("bro", broConfig); parserConfigurations.updateSensorParserConfig("snort", snortConfig); parserConfigurations.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfigString, JSONUtils.MAP_SUPPLIER)); parserRunner = new ParserRunnerImpl(new HashSet<>(Arrays.asList("bro", "snort"))); broParser = mock(MessageParser.class); snortParser = mock(MessageParser.class); stellarFilter = mock(StellarFilter.class); mockStatic(ReflectionUtils.class); mockStatic(Filters.class); when(ReflectionUtils.createInstance("org.apache.metron.parsers.bro.BasicBroParser")).thenReturn(broParser); when(ReflectionUtils.createInstance("org.apache.metron.parsers.snort.BasicSnortParser")).thenReturn(snortParser); when(Filters.get("org.apache.metron.parsers.filters.StellarFilter", broConfig.getParserConfig())) .thenReturn(stellarFilter); }
.withErrorFields(new HashSet<>(Arrays.asList("ip_src_addr", "ip_dst_addr"))); when(stellarFilter.emit(expectedOutput, parserRunner.getStellarContext())).thenReturn(true); when(broParser.validate(expectedOutput)).thenReturn(true); parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {{ put("bro", new ParserComponent(broParser, stellarFilter)); }}); Optional<ParserRunnerImpl.ProcessResult> processResult = parserRunner.processMessage("bro", inputMessage, rawMessage, broParser, parserConfigurations);
verify(parserRunner, times(1)).init(any(Supplier.class), eq(stellarContext)); verify(yafConfig, times(1)).init(); Map<String, String> topicToSensorMap = parserBolt.getTopicToSensorMap();
public ParserDriver(String sensorType, String parserConfig, String globalConfig) throws IOException { SensorParserConfig sensorParserConfig = SensorParserConfig.fromBytes(parserConfig.getBytes()); this.sensorType = sensorType == null ? sensorParserConfig.getSensorTopic() : sensorType; config = new ParserConfigurations(); config.updateSensorParserConfig(this.sensorType, SensorParserConfig.fromBytes(parserConfig.getBytes())); config.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER)); parserRunner = new ParserRunnerImpl(new HashSet<String>() {{ add(sensorType); }}); }