/** * Take raw data and convert it to an optional list of messages. * @param parseMessage the raw bytes of the message * @return If null is returned, this is treated as an empty list. */ @Deprecated default Optional<List<T>> parseOptional(byte[] parseMessage) { return Optional.ofNullable(parse(parseMessage)); }
/** * Take raw data and convert it to messages. Each raw message may produce multiple messages and therefore * multiple errors. A {@link MessageParserResult} is returned, which will have both the messages produced * and the errors. * @param parseMessage the raw bytes of the message * @return Optional of {@link MessageParserResult} */ default Optional<MessageParserResult<T>> parseOptionalResult(byte[] parseMessage) { Optional<MessageParserResult<T>> result = Optional.empty(); try { Optional<List<T>> optionalMessages = parseOptional(parseMessage); if (optionalMessages.isPresent()) { result = Optional.of(new DefaultMessageParserResult<>(optionalMessages.get())); } } catch (Throwable t) { return Optional.of(new DefaultMessageParserResult<>(t)); } return result; }
@Test public void testNotNullable() throws Exception { MessageParser<JSONObject> parser = new TestMessageParser() { @Override public List<JSONObject> parse(byte[] rawMessage) { return new ArrayList<>(); } }; Assert.assertNotNull(parser.parseOptionalResult(null)); Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult(null); Assert.assertTrue(ret.isPresent()); Assert.assertEquals(0, ret.get().getMessages().size()); }
.put(MetronRestConstants.GROK_PATH_KEY, new Path(temporaryGrokPath, name).toString()); parser.configure(sensorParserConfig.getParserConfig()); parser.init(); Optional<MessageParserResult<JSONObject>> result = parser.parseOptionalResult(parseMessageRequest.getSampleData().getBytes()); if (!result.isPresent()) { throw new RestException("Unknown error parsing sample data"); if (result.get().getMasterThrowable().isPresent()) { throw new RestException("Error parsing sample data",result.get().getMasterThrowable().get()); if (result.get().getMessages().isEmpty()) { throw new RestException("No results from parsing sample data"); grokService.deleteTemporary(); return result.get().getMessages().get(0);
@Test public void shouldPopulateMessagesOnProcessMessage() { JSONObject inputMessage = new JSONObject(); inputMessage.put("guid", "guid"); inputMessage.put("ip_src_addr", "192.168.1.1"); inputMessage.put("ip_dst_addr", "192.168.1.2"); inputMessage.put("field1", "value"); RawMessage rawMessage = new RawMessage("raw_message".getBytes(), new HashMap<>()); JSONObject expectedOutput = new JSONObject(); expectedOutput.put("guid", "guid"); expectedOutput.put("source.type", "bro"); expectedOutput.put("ip_src_addr", "192.168.1.1"); expectedOutput.put("ip_dst_addr", "192.168.1.2"); when(stellarFilter.emit(expectedOutput, parserRunner.getStellarContext())).thenReturn(true); when(broParser.validate(expectedOutput)).thenReturn(true); parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {{ put("bro", new ParserComponent(broParser, stellarFilter)); }}); Optional<ParserRunnerImpl.ProcessResult> processResult = parserRunner.processMessage("bro", inputMessage, rawMessage, broParser, parserConfigurations); Assert.assertTrue(processResult.isPresent()); Assert.assertFalse(processResult.get().isError()); Assert.assertEquals(expectedOutput, processResult.get().getMessage()); }
@Test public void testParseException() { MessageParser<JSONObject> parser = new TestMessageParser() { @Override public List<JSONObject> parse(byte[] rawMessage) { throw new RuntimeException("parse exception"); } }; Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult("message".getBytes()); Assert.assertTrue(ret.isPresent()); Assert.assertTrue(ret.get().getMasterThrowable().isPresent()); Assert.assertEquals("parse exception", ret.get().getMasterThrowable().get().getMessage()); }
@Test public void testReadMultiLine() throws Exception { Syslog3164Parser parser = new Syslog3164Parser(); Map<String, Object> config = new HashMap<>(); parser.configure(config); StringBuilder builder = new StringBuilder(); builder .append(SYSLOG_LINE_ALL) .append("\n") .append(SYSLOG_LINE_MISSING) .append("\n") .append(SYSLOG_LINE_ALL); Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(builder.toString().getBytes()); Assert.assertNotNull(resultOptional); Assert.assertTrue(resultOptional.isPresent()); List<JSONObject> parsedList = resultOptional.get().getMessages(); Assert.assertEquals(3,parsedList.size()); }
@Test public void testReadMultiLineWithErrors() throws Exception { Syslog3164Parser parser = new Syslog3164Parser(); Map<String, Object> config = new HashMap<>(); parser.configure(config); StringBuilder builder = new StringBuilder(); builder .append("HEREWEGO!!!!\n") .append(SYSLOG_LINE_ALL) .append("\n") .append(SYSLOG_LINE_MISSING) .append("\n") .append("BOOM!\n") .append(SYSLOG_LINE_ALL) .append("\nOHMY!"); Optional<MessageParserResult<JSONObject>> output = parser.parseOptionalResult(builder.toString().getBytes()); Assert.assertTrue(output.isPresent()); Assert.assertEquals(3,output.get().getMessages().size()); Assert.assertEquals(3,output.get().getMessageThrowables().size()); } }
@Test public void testSingleQueryFilter() throws Exception { { Map<String, Object> config = new HashMap<String, Object>() {{ put("filter.query", "exists(foo)"); }}; MessageFilter<JSONObject> filter = Filters.get(Filters.STELLAR.name(), config); Assert.assertTrue(filter.emit(new JSONObject(ImmutableMap.of("foo", 1)), Context.EMPTY_CONTEXT())); Assert.assertFalse(filter.emit(new JSONObject(ImmutableMap.of("bar", 1)), Context.EMPTY_CONTEXT())); } }
@Test public void testNullable() throws Exception { MessageParser parser = new TestMessageParser() { @Override public List<JSONObject> parse(byte[] rawMessage) { return null; } }; Assert.assertNotNull(parser.parseOptionalResult(null)); Assert.assertFalse(parser.parseOptionalResult(null).isPresent()); }
public static MessageFilter<JSONObject> get(String filterName, Map<String, Object> config) { if(filterName == null || filterName.trim().isEmpty()) { return null; } Class<? extends MessageFilter> filterClass; try { Filters f = Filters.valueOf(filterName); filterClass = f.clazz; } catch(Exception ex) { try { filterClass = (Class<? extends MessageFilter>) Class.forName(filterName); } catch (ClassNotFoundException e) { throw new IllegalStateException("Unable to find class " + filterName, e); } } if(filterClass != null) { MessageFilter<JSONObject> filter = ReflectionUtils.createInstance(filterClass); filter.configure(config); return filter; } return null; } }
@Test public void testParse() { JSONObject message = new JSONObject(); MessageParser<JSONObject> parser = new TestMessageParser() { @Override public List<JSONObject> parse(byte[] rawMessage) { return Collections.singletonList(message); } }; Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult("message".getBytes()); Assert.assertTrue(ret.isPresent()); Assert.assertEquals(1, ret.get().getMessages().size()); Assert.assertEquals(message, ret.get().getMessages().get(0)); }
@Test public void shouldReturnMetronErrorOnInvalidMessage() { JSONObject inputMessage = new JSONObject(); inputMessage.put("guid", "guid"); RawMessage rawMessage = new RawMessage("raw_message".getBytes(), new HashMap<>()); JSONObject expectedOutput = new JSONObject(); expectedOutput.put("guid", "guid"); expectedOutput.put("source.type", "bro"); MetronError expectedMetronError = new MetronError() .withErrorType(Constants.ErrorType.PARSER_INVALID) .withSensorType(Collections.singleton("bro")) .addRawMessage(inputMessage); when(stellarFilter.emit(expectedOutput, parserRunner.getStellarContext())).thenReturn(true); when(broParser.validate(expectedOutput)).thenReturn(false); parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {{ put("bro", new ParserComponent(broParser, stellarFilter)); }}); Optional<ParserRunnerImpl.ProcessResult> processResult = parserRunner.processMessage("bro", inputMessage, rawMessage, broParser, parserConfigurations); Assert.assertTrue(processResult.isPresent()); Assert.assertTrue(processResult.get().isError()); Assert.assertEquals(expectedMetronError, processResult.get().getError()); }
@Test public void testParseOptionalException() { MessageParser<JSONObject> parser = new TestMessageParser() { @Override public Optional<List<JSONObject>> parseOptional(byte[] rawMessage) { throw new RuntimeException("parse exception"); } }; Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult("message".getBytes()); Assert.assertTrue(ret.isPresent()); Assert.assertTrue(ret.get().getMasterThrowable().isPresent()); Assert.assertEquals("parse exception", ret.get().getMasterThrowable().get().getMessage()); }
@Test public void testReadMultiLine() throws Exception { Syslog5424Parser parser = new Syslog5424Parser(); Map<String, Object> config = new HashMap<>(); config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.DASH.name()); parser.configure(config); StringBuilder builder = new StringBuilder(); builder .append(SYSLOG_LINE_ALL) .append("\n") .append(SYSLOG_LINE_MISSING) .append("\n") .append(SYSLOG_LINE_ALL); Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(builder.toString().getBytes()); Assert.assertNotNull(resultOptional); Assert.assertTrue(resultOptional.isPresent()); List<JSONObject> parsedList = resultOptional.get().getMessages(); Assert.assertEquals(3,parsedList.size()); }
@Test public void testSingleQueryFilter() throws Exception { { Map<String, Object> config = new HashMap<String, Object>() {{ put("filter.query", "exists(foo)"); }}; MessageFilter<JSONObject> filter = Filters.get(Filters.STELLAR.name(), config); Assert.assertTrue(filter.emit(new JSONObject(ImmutableMap.of("foo", 1)), Context.EMPTY_CONTEXT())); Assert.assertFalse(filter.emit(new JSONObject(ImmutableMap.of("bar", 1)), Context.EMPTY_CONTEXT())); } }
@Test public void testParseOptional() { JSONObject message = new JSONObject(); MessageParser<JSONObject> parser = new TestMessageParser() { @Override public Optional<List<JSONObject>> parseOptional(byte[] rawMessage) { return Optional.of(Collections.singletonList(message)); } }; Optional<MessageParserResult<JSONObject>> ret = parser.parseOptionalResult("message".getBytes()); Assert.assertTrue(ret.isPresent()); Assert.assertEquals(1, ret.get().getMessages().size()); Assert.assertEquals(message, ret.get().getMessages().get(0)); }
@Test public void testParseRBMLine() throws Exception { //Set up parser, parse message GrokWebSphereParser parser = new GrokWebSphereParser(); parser.configure(parserConfig); String testString = "<131>Apr 15 17:36:35 ROBXML3QRS [0x80800018][auth][error] rbm(RBM-Settings): " + "trans(3502888135)[request] gtid(3502888135): RBM: Resource access denied."; Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes()); Assert.assertNotNull(resultOptional); Assert.assertTrue(resultOptional.isPresent()); List<JSONObject> result = resultOptional.get().getMessages(); JSONObject parsedJSON = result.get(0); long expectedTimestamp = ZonedDateTime.of(Year.now(UTC).getValue(), 4, 15, 17, 36, 35, 0, UTC).toInstant().toEpochMilli(); //Compare fields assertEquals(131, parsedJSON.get("priority")); assertEquals(expectedTimestamp, parsedJSON.get("timestamp")); assertEquals("ROBXML3QRS", parsedJSON.get("hostname")); assertEquals("0x80800018", parsedJSON.get("event_code")); assertEquals("auth", parsedJSON.get("event_type")); assertEquals("error", parsedJSON.get("severity")); assertEquals("rbm", parsedJSON.get("process")); assertEquals("trans(3502888135)[request] gtid(3502888135): RBM: Resource access denied.", parsedJSON.get("message")); }
@Test public void testParseMalformedLogoutLine() throws Exception { //Set up parser, attempt to parse malformed message GrokWebSphereParser parser = new GrokWebSphereParser(); parser.configure(parserConfig); String testString = "<134>Apr 15 18:02:27 PHIXML3RWD [0x81000019][auth][info] [14.122.2.201: " + "User 'hjpotter' logged out from 'default."; Optional<MessageParserResult<JSONObject>> resultOptional = parser.parseOptionalResult(testString.getBytes()); Assert.assertNotNull(resultOptional); Assert.assertTrue(resultOptional.isPresent()); List<JSONObject> result = resultOptional.get().getMessages(); JSONObject parsedJSON = result.get(0); long expectedTimestamp = ZonedDateTime.of(Year.now(UTC).getValue(), 4, 15, 18, 2, 27, 0, UTC).toInstant().toEpochMilli(); //Compare fields assertEquals(134, parsedJSON.get("priority")); assertEquals(expectedTimestamp, parsedJSON.get("timestamp")); assertEquals("PHIXML3RWD", parsedJSON.get("hostname")); assertEquals("0x81000019", parsedJSON.get("event_code")); assertEquals("auth", parsedJSON.get("event_type")); assertEquals("info", parsedJSON.get("severity")); assertEquals(null, parsedJSON.get("ip_src_addr")); assertEquals(null, parsedJSON.get("username")); assertEquals(null, parsedJSON.get("security_domain")); }