How to use

ParserRunnerImpl

in

org.apache.metron.parsers

private List<JSONObject> doParse(List<String> messages) {
  // initialize
  HashSet<String> sensorTypes = new HashSet<>();
  sensorTypes.add(sensorType);
  ParserRunnerImpl runner = new ParserRunnerImpl(sensorTypes);
  runner.init(() -> parserConfigurations, context);
  // parse each message
  List<ParserRunnerResults<JSONObject>> results = messages
      .stream()
      .map(str -> str.getBytes())
      .map(bytes -> DEFAULT.get(emptyMap(), bytes, false, emptyMap()))
      .map(msg -> runner.execute(sensorType, msg, parserConfigurations))
      .collect(Collectors.toList());
  // aggregate both successes and errors into a list that can be returned
  List<JSONObject> successes = results
      .stream()
      .flatMap(result -> result.getMessages().stream())
      .collect(Collectors.toList());
  successCount += successes.size();
  List<JSONObject> errors = results
      .stream()
      .flatMap(result -> result.getErrors().stream())
      .map(err -> err.getJSONObject())
      .collect(Collectors.toList());
  errorCount += errors.size();
  // return a list of both successes and errors
  successes.addAll(errors);
  return successes;
}

@Override
public void init(Supplier<ParserConfigurations> parserConfigSupplier, Context stellarContext) {
 if (parserConfigSupplier == null) {
  throw new IllegalStateException("A parser config supplier must be set before initializing the ParserRunner.");
 }
 if (stellarContext == null) {
  throw new IllegalStateException("A stellar context must be set before initializing the ParserRunner.");
 }
 this.stellarContext = stellarContext;
 initializeParsers(parserConfigSupplier);
}

@Test
public void initShouldThrowExceptionOnMissingSensorParserConfig() {
 exception.expect(IllegalStateException.class);
 exception.expectMessage("Could not initialize parsers.  Cannot find configuration for sensor test.");
 parserRunner = new ParserRunnerImpl(new HashSet<String>() {{
  add("test");
 }});
 parserRunner.init(() -> parserConfigurations, mock(Context.class));
}

@Test
public void executeShouldThrowExceptionOnMissingSensorParserConfig() {
 exception.expect(IllegalStateException.class);
 exception.expectMessage("Could not execute parser.  Cannot find configuration for sensor test.");
 parserRunner = new ParserRunnerImpl(new HashSet<String>() {{
  add("test");
 }});
 parserRunner.execute("test", mock(RawMessage.class), parserConfigurations);
}

@Test
public void shouldExecuteWithMasterThrowable() {
 parserRunner = spy(parserRunner);
 RawMessage rawMessage = new RawMessage("raw_message".getBytes(), new HashMap<>());
 Throwable masterThrowable = mock(Throwable.class);
 MessageParserResult<JSONObject> messageParserResult = new DefaultMessageParserResult<>(masterThrowable);
 when(broParser.parseOptionalResult(rawMessage.getMessage())).thenReturn(Optional.of(messageParserResult));
 parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {{
  put("bro", new ParserComponent(broParser, stellarFilter));
 }});
 ParserRunnerResults<JSONObject> parserRunnerResults = parserRunner.execute("bro", rawMessage, parserConfigurations);
 verify(parserRunner, times(0))
     .processMessage(any(), any(), any(), any(), any());
 MetronError expectedError = new MetronError()
     .withErrorType(Constants.ErrorType.PARSER_ERROR)
     .withThrowable(masterThrowable)
     .withSensorType(Collections.singleton("bro"))
     .addRawMessage(rawMessage.getMessage());
 Assert.assertEquals(1, parserRunnerResults.getErrors().size());
 Assert.assertTrue(parserRunnerResults.getErrors().contains(expectedError));
}

@Test
public void shouldPopulateMessagesOnProcessMessage() {
 JSONObject inputMessage = new JSONObject();
 inputMessage.put("guid", "guid");
 inputMessage.put("ip_src_addr", "192.168.1.1");
 inputMessage.put("ip_dst_addr", "192.168.1.2");
 inputMessage.put("field1", "value");
 RawMessage rawMessage = new RawMessage("raw_message".getBytes(), new HashMap<>());
 JSONObject expectedOutput  = new JSONObject();
 expectedOutput.put("guid", "guid");
 expectedOutput.put("source.type", "bro");
 expectedOutput.put("ip_src_addr", "192.168.1.1");
 expectedOutput.put("ip_dst_addr", "192.168.1.2");
 when(stellarFilter.emit(expectedOutput, parserRunner.getStellarContext())).thenReturn(true);
 when(broParser.validate(expectedOutput)).thenReturn(true);
 parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {{
  put("bro", new ParserComponent(broParser, stellarFilter));
 }});
 Optional<ParserRunnerImpl.ProcessResult> processResult = parserRunner.processMessage("bro", inputMessage, rawMessage, broParser, parserConfigurations);
 Assert.assertTrue(processResult.isPresent());
 Assert.assertFalse(processResult.get().isError());
 Assert.assertEquals(expectedOutput, processResult.get().getMessage());
}

Map<String, Object> snortParserConfig = parserConfigurations.getSensorParserConfig("snort").getParserConfig();
parserRunner.init(() -> parserConfigurations, stellarContext);
 Assert.assertEquals(stellarContext, parserRunner.getStellarContext());
Map<String, ParserComponent> sensorToParserComponentMap = parserRunner.getSensorToParserComponentMap();

@Test
public void shouldThrowExceptionOnEmptyParserSupplier() {
 exception.expect(IllegalStateException.class);
 exception.expectMessage("A parser config supplier must be set before initializing the ParserRunner.");
 parserRunner.init(null, null);
}

return new ParserBolt(zookeeperUrl, new ParserRunnerImpl(new HashSet<>(sensorTypeToParserConfig.keySet())), writerConfigs);

doThrow(new IllegalStateException("parserRunner.execute failed")).when(parserRunner).execute(eq("yaf"), any(), eq(parserConfigurations));

);
message.put(Constants.SENSOR_TYPE, sensorType);
applyFieldTransformations(message, rawMessage, sensorParserConfig);
if (!message.containsKey(Constants.GUID)) {
 message.put(Constants.GUID, UUID.randomUUID().toString());
 List<FieldValidator> failedValidators = null;
 if (!isInvalid) {
  failedValidators = getFailedValidators(message, parserConfigurations);
  isInvalid = !failedValidators.isEmpty();

Optional<ProcessResult> processResult = processMessage(sensorType, message, rawMessage, parser, parserConfigurations);
if (processResult.isPresent()) {
 if (processResult.get().isError()) {

when(processedErrorResult.getError()).thenReturn(processedError);
doReturn(Optional.of(processedMessageResult)).when(parserRunner)
    .processMessage("bro", parsedMessage1, rawMessage, broParser, parserConfigurations);
doReturn(Optional.of(processedErrorResult)).when(parserRunner)
    .processMessage("bro", parsedMessage2, rawMessage, broParser, parserConfigurations);
    .addRawMessage(rawMessage2);
parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {{
 put("bro", new ParserComponent(broParser, stellarFilter));
}});
ParserRunnerResults<JSONObject> parserRunnerResults = parserRunner.execute("bro", rawMessage, parserConfigurations);

@Test
public void shouldReturnMetronErrorOnInvalidMessage() {
 JSONObject inputMessage = new JSONObject();
 inputMessage.put("guid", "guid");
 RawMessage rawMessage = new RawMessage("raw_message".getBytes(), new HashMap<>());
 JSONObject expectedOutput  = new JSONObject();
 expectedOutput.put("guid", "guid");
 expectedOutput.put("source.type", "bro");
 MetronError expectedMetronError = new MetronError()
     .withErrorType(Constants.ErrorType.PARSER_INVALID)
     .withSensorType(Collections.singleton("bro"))
     .addRawMessage(inputMessage);
 when(stellarFilter.emit(expectedOutput, parserRunner.getStellarContext())).thenReturn(true);
 when(broParser.validate(expectedOutput)).thenReturn(false);
 parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {{
  put("bro", new ParserComponent(broParser, stellarFilter));
 }});
 Optional<ParserRunnerImpl.ProcessResult> processResult = parserRunner.processMessage("bro", inputMessage, rawMessage, broParser, parserConfigurations);
 Assert.assertTrue(processResult.isPresent());
 Assert.assertTrue(processResult.get().isError());
 Assert.assertEquals(expectedMetronError, processResult.get().getError());
}

@Test
public void shouldThrowExceptionOnEmptyStellarContext() {
 exception.expect(IllegalStateException.class);
 exception.expectMessage("A stellar context must be set before initializing the ParserRunner.");
 parserRunner.init(() -> parserConfigurations, null);
}

@Before
public void setup() throws IOException {
 parserConfigurations = new ParserConfigurations();
 SensorParserConfig broConfig = SensorParserConfig.fromBytes(broConfigString.getBytes());
 SensorParserConfig snortConfig = SensorParserConfig.fromBytes(snortConfigString.getBytes());
 parserConfigurations.updateSensorParserConfig("bro", broConfig);
 parserConfigurations.updateSensorParserConfig("snort", snortConfig);
 parserConfigurations.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfigString, JSONUtils.MAP_SUPPLIER));
 parserRunner = new ParserRunnerImpl(new HashSet<>(Arrays.asList("bro", "snort")));
 broParser = mock(MessageParser.class);
 snortParser = mock(MessageParser.class);
 stellarFilter = mock(StellarFilter.class);
 mockStatic(ReflectionUtils.class);
 mockStatic(Filters.class);
 when(ReflectionUtils.createInstance("org.apache.metron.parsers.bro.BasicBroParser")).thenReturn(broParser);
 when(ReflectionUtils.createInstance("org.apache.metron.parsers.snort.BasicSnortParser")).thenReturn(snortParser);
 when(Filters.get("org.apache.metron.parsers.filters.StellarFilter", broConfig.getParserConfig()))
     .thenReturn(stellarFilter);
}

    .withErrorFields(new HashSet<>(Arrays.asList("ip_src_addr", "ip_dst_addr")));
when(stellarFilter.emit(expectedOutput, parserRunner.getStellarContext())).thenReturn(true);
when(broParser.validate(expectedOutput)).thenReturn(true);
parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {{
 put("bro", new ParserComponent(broParser, stellarFilter));
}});
Optional<ParserRunnerImpl.ProcessResult> processResult = parserRunner.processMessage("bro", inputMessage, rawMessage, broParser, parserConfigurations);

verify(parserRunner, times(1)).init(any(Supplier.class), eq(stellarContext));
verify(yafConfig, times(1)).init();
Map<String, String> topicToSensorMap = parserBolt.getTopicToSensorMap();

public ParserDriver(String sensorType, String parserConfig, String globalConfig)
  throws IOException {
 SensorParserConfig sensorParserConfig = SensorParserConfig.fromBytes(parserConfig.getBytes());
 this.sensorType = sensorType == null ? sensorParserConfig.getSensorTopic() : sensorType;
 config = new ParserConfigurations();
 config.updateSensorParserConfig(this.sensorType,
   SensorParserConfig.fromBytes(parserConfig.getBytes()));
 config.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER));
 parserRunner = new ParserRunnerImpl(new HashSet<String>() {{
  add(sensorType);
 }});
}