@SuppressWarnings("unchecked") @Override public Optional<MessageParserResult<JSONObject>> parseOptionalResult(byte[] rawMessage) { if (grok == null) { init(); } if (multiLine) { return parseMultiLine(rawMessage); } return parseSingleLine(rawMessage); }
@Override public void init() { grok = new Grok(); try { InputStream commonInputStream = openInputStream(patternsCommonDir); LOG.debug("Grok parser loading common patterns from: {}", patternsCommonDir); if (commonInputStream == null) { throw new RuntimeException( "Unable to initialize grok parser: Unable to load " + patternsCommonDir + " from either classpath or HDFS"); } grok.addPatternFromReader(new InputStreamReader(commonInputStream)); LOG.debug("Loading parser-specific patterns from: {}", grokPath); InputStream patterInputStream = openInputStream(grokPath); if (patterInputStream == null) { throw new RuntimeException("Grok parser unable to initialize grok parser: Unable to load " + grokPath + " from either classpath or HDFS"); } grok.addPatternFromReader(new InputStreamReader(patterInputStream)); if (LOG.isDebugEnabled()) { LOG.debug("Grok parser set the following grok expression: {}", grok.getNamedRegexCollectionById(patternLabel)); } String grokPattern = "%{" + patternLabel + "}"; grok.compile(grokPattern); LOG.debug("Compiled grok pattern {}", grokPattern); } catch (Throwable e) { LOG.error(e.getMessage(), e); throw new RuntimeException("Grok parser Error: " + e.getMessage(), e); } }
@Test public void test() throws IOException, ParseException { Map<String, Object> parserConfig = new HashMap<>(); parserConfig.put("grokPath", getGrokPath()); parserConfig.put("patternLabel", getGrokPatternLabel()); parserConfig.put("timestampField", getTimestampField()); parserConfig.put("dateFormat", getDateFormat()); parserConfig.put("timeFields", getTimeFields()); GrokParser grokParser = new GrokParser(); grokParser.configure(parserConfig); grokParser.init(); JSONParser jsonParser = new JSONParser(); Map<String,String> testData = getTestData(); for( Map.Entry<String,String> e : testData.entrySet() ){ JSONObject expected = (JSONObject) jsonParser.parse(e.getValue()); byte[] rawMessage = e.getKey().getBytes(); Optional<MessageParserResult<JSONObject>> resultOptional = grokParser.parseOptionalResult(rawMessage); Assert.assertNotNull(resultOptional); Assert.assertTrue(resultOptional.isPresent()); List<JSONObject> parsedList = resultOptional.get().getMessages(); Assert.assertEquals(1, parsedList.size()); compare(expected, parsedList.get(0)); } }
/** * Test that if a byte[] with multiple lines of log is passed in * it will be parsed into the correct number of messages. * @throws IOException if we can't read from disk * @throws ParseException if we can't parse */ @Test(expected = RuntimeException.class) @SuppressWarnings("unchecked") public void testLegacyInterfaceThrowsOneExceptionWithMultiline() throws IOException, ParseException { Map<String, Object> parserConfig = new HashMap<>(); parserConfig.put("grokPath", getGrokPath()); parserConfig.put("patternLabel", getGrokPatternLabel()); parserConfig.put("timestampField", getTimestampField()); parserConfig.put("dateFormat", getDateFormat()); parserConfig.put("timeFields", getTimeFields()); parserConfig.put("multiLine",getMultiLine()); GrokParser grokParser = new GrokParser(); grokParser.configure(parserConfig); grokParser.init(); JSONParser jsonParser = new JSONParser(); Map<String, String> testData = getTestData(); for (Map.Entry<String, String> e : testData.entrySet()) { byte[] rawMessage = e.getKey().getBytes(); List<JSONObject> parsedList = grokParser.parse(rawMessage); } }
String fieldValue = (String) message.get(timeField); if (fieldValue != null) { message.put(timeField, toEpoch(fieldValue)); message.put(Constants.Fields.TIMESTAMP.getName(), formatTimestamp(message.get(timestampField))); postParse(message); messages.add(message); LOG.debug("Grok parser parsed message: {}", message);
/** * Test that if a byte[] with multiple lines of log is passed in * it will be parsed into the correct number of messages. * @throws IOException if we can't read from disk * @throws ParseException if we can't parse */ @Test @SuppressWarnings("unchecked") public void testLegacyInterfaceReturnsMultiline() throws IOException, ParseException { Map<String, Object> parserConfig = new HashMap<>(); parserConfig.put("grokPath", getGrokPath()); parserConfig.put("patternLabel", getGrokPatternLabel()); parserConfig.put("timestampField", getTimestampField()); parserConfig.put("dateFormat", getDateFormat()); parserConfig.put("timeFields", getTimeFields()); parserConfig.put("multiLine", getMultiLine()); GrokParser grokParser = new GrokParser(); grokParser.configure(parserConfig); grokParser.init(); JSONParser jsonParser = new JSONParser(); Map<String, String> testData = getTestData(); for (Map.Entry<String, String> e : testData.entrySet()) { byte[] rawMessage = e.getKey().getBytes(); Optional<MessageParserResult<JSONObject>> resultOptional = grokParser.parseOptionalResult(rawMessage); Assert.assertNotNull(resultOptional); Assert.assertTrue(resultOptional.isPresent()); List<JSONObject> parsedList = resultOptional.get().getMessages(); Assert.assertEquals(10, parsedList.size()); } }
String fieldValue = (String) message.get(timeField); if (fieldValue != null) { message.put(timeField, toEpoch(fieldValue)); message.put(Constants.Fields.TIMESTAMP.getName(), formatTimestamp(message.get(timestampField))); postParse(message); messages.add(message); LOG.debug("Grok parser parsed message: {}", message);
parserConfig.put("multiLine",getMultiLine()); GrokParser grokParser = new GrokParser(); grokParser.configure(parserConfig); grokParser.init(); for (Map.Entry<String, String> e : testData.entrySet()) { byte[] rawMessage = e.getKey().getBytes(); Optional<MessageParserResult<JSONObject>> resultOptional = grokParser.parseOptionalResult(rawMessage); Assert.assertTrue(resultOptional.isPresent()); Optional<Throwable> throwableOptional = resultOptional.get().getMasterThrowable();
parserConfig.put("multiLine", getMultiLine()); GrokParser grokParser = new GrokParser(); grokParser.configure(parserConfig); grokParser.init(); for (Map.Entry<String, String> e : testData.entrySet()) { byte[] rawMessage = e.getKey().getBytes(); Optional<MessageParserResult<JSONObject>> resultOptional = grokParser.parseOptionalResult(rawMessage); Assert.assertTrue(resultOptional.isPresent()); Optional<Throwable> throwableOptional = resultOptional.get().getMasterThrowable();