public ProcessorResult<List<byte[]>> run(Iterable<byte[]> in) { ShimParserBolt bolt = new ShimParserBolt(new ArrayList<>()); byte[] b = SerializationUtils.serialize(bolt); ShimParserBolt b2 = (ShimParserBolt) SerializationUtils.deserialize(b); OutputCollector collector = mock(OutputCollector.class); bolt.prepare(null, null, collector); for(byte[] record : in) { bolt.execute(toTuple(record)); } return bolt.getResults(); }
public ShimParserBolt(List<byte[]> output) { super(null, parserRunner, Collections.singletonMap(sensorType, new WriterHandler(new CollectingWriter(output)))); this.output = output; }
@Override List<ParserValidation> getValidations() { return new ArrayList<ParserValidation>() {{ add(new SampleDataValidation()); }}; } }
@Test public void testCiscoPixEnvelopingCisco302020() throws Exception { ParserDriver syslogDriver = new StormParserDriver("ciscoPix", ciscoPixSyslogConfig, "{}"); ParserDriver driver = new StormParserDriver("cisco302020", cisco302020Config, "{}"); super.testCiscoPixEnvelopingCisco302020(syslogDriver, driver); } }
public void runTest(ParserDriver driver) throws Exception { String sensorType = driver.getSensorType(); inputMessages = TestUtils.readSampleData(SampleDataUtils.getSampleDataPath("..", sensorType, TestDataType.RAW)); ProcessorResult<List<byte[]>> result = driver.run(inputMessages); List<byte[]> outputMessages = result.getResult(); StringBuffer buffer = new StringBuffer(); if (result.failed()){ result.getBadResults(buffer); buffer.append(String.format("%d Valid Messages Processed", outputMessages.size())).append("\n"); dumpParsedMessages(outputMessages,buffer); Assert.fail(buffer.toString()); } else { List<ParserValidation> validations = getValidations(); if (validations == null || validations.isEmpty()) { buffer.append("No validations configured for sensorType ").append(sensorType).append(". Dumping parsed messages").append("\n"); dumpParsedMessages(outputMessages,buffer); Assert.fail(buffer.toString()); } else { for (ParserValidation validation : validations) { System.out.println("Running " + validation.getName() + " on sensorType " + sensorType); validation.validate(sensorType, outputMessages); } } } }
public void testCiscoPixEnvelopingCisco302020(ParserDriver syslogDriver, ParserDriver driver) throws Exception { byte[] envelopedData = null; String inputRecord = "Mar 29 2004 09:54:18: %PIX-6-302005: Built UDP connection for faddr 198.207.223.240/53337 gaddr 10.0.0.187/53 laddr 192.168.0.2/53"; ProcessorResult<List<byte[]>> syslogResult = syslogDriver.run(ImmutableList.of(inputRecord.getBytes())); Assert.assertFalse(syslogResult.failed()); List<byte[]> syslogResultList = syslogResult.getResult(); envelopedData = syslogResultList.get(0); ProcessorResult<List<byte[]>> results = driver.run(ImmutableList.of(envelopedData)); Assert.assertFalse(results.failed()); List<byte[]> resultList = results.getResult(); Assert.assertEquals(1, resultList.size()); Map<String, Object> result = JSONUtils.INSTANCE.load(new String(resultList.get(0)), JSONUtils.MAP_SUPPLIER); Assert.assertEquals("UDP", result.get("protocol")); Assert.assertTrue((long) result.get("timestamp") > 1000); } }
public ParserDriver(String sensorType, String parserConfig, String globalConfig) throws IOException { SensorParserConfig sensorParserConfig = SensorParserConfig.fromBytes(parserConfig.getBytes()); this.sensorType = sensorType == null ? sensorParserConfig.getSensorTopic() : sensorType; config = new ParserConfigurations(); config.updateSensorParserConfig(this.sensorType, SensorParserConfig.fromBytes(parserConfig.getBytes())); config.updateGlobalConfig(JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER)); parserRunner = new ParserRunnerImpl(new HashSet<String>() {{ add(sensorType); }}); }
@Test public void testEnvelopedData() throws IOException { ParserDriver driver = new StormParserDriver("test", parserConfig_default, "{}"); super.testEnvelopedData(driver); }
public void testEnvelopedData(ParserDriver driver) throws IOException { Map<String, Object> inputRecord = new HashMap<String, Object>() {{ put(Constants.Fields.ORIGINAL.getName(), "real_original_string"); put("data", "field1_val,100"); put("metadata_field", "metadata_val"); }}; ProcessorResult<List<byte[]>> results = driver.run(ImmutableList.of(JSONUtils.INSTANCE.toJSONPretty(inputRecord))); Assert.assertFalse(results.failed()); List<byte[]> resultList = results.getResult(); Assert.assertEquals(1, resultList.size()); Map<String, Object> outputRecord = JSONUtils.INSTANCE.load(new String(resultList.get(0)), JSONUtils.MAP_SUPPLIER); Assert.assertEquals("field1_val", outputRecord.get("field1")); Assert.assertEquals(inputRecord.get(Constants.Fields.ORIGINAL.getName()), outputRecord.get(Constants.Fields.ORIGINAL.getName())); Assert.assertEquals(inputRecord.get(MetadataUtil.METADATA_PREFIX + ".metadata_field"), outputRecord.get("metadata_field")); }
@Test public void testEnvelopedData_noMergeMetadata() throws IOException { ParserDriver driver = new StormParserDriver("test", parserConfig_nomerge, "{}"); super.testEnvelopedData_noMergeMetadata(driver); }
public void testEnvelopedData_noMergeMetadata(ParserDriver driver) throws IOException { Map<String, Object> inputRecord = new HashMap<String, Object>() {{ put(Constants.Fields.ORIGINAL.getName(), "real_original_string"); put("data", "field1_val,100"); put("metadata_field", "metadata_val"); }}; ProcessorResult<List<byte[]>> results = driver.run(ImmutableList.of(JSONUtils.INSTANCE.toJSONPretty(inputRecord))); Assert.assertFalse(results.failed()); List<byte[]> resultList = results.getResult(); Assert.assertEquals(1, resultList.size()); Map<String, Object> outputRecord = JSONUtils.INSTANCE.load(new String(resultList.get(0)), JSONUtils.MAP_SUPPLIER); Assert.assertEquals("field1_val", outputRecord.get("field1")); Assert.assertEquals(inputRecord.get(Constants.Fields.ORIGINAL.getName()), outputRecord.get(Constants.Fields.ORIGINAL.getName())); Assert.assertFalse(outputRecord.containsKey(MetadataUtil.METADATA_PREFIX + ".metadata_field")); }
@Test public void testEnvelopedData_withMetadataPrefix() throws IOException { ParserDriver driver = new StormParserDriver("test", parserConfig_withPrefix, "{}"); super.testEnvelopedData_withMetadataPrefix(driver); }
public void testEnvelopedData_withMetadataPrefix(ParserDriver driver) throws IOException { Map<String, Object> inputRecord = new HashMap<String, Object>() {{ put(Constants.Fields.ORIGINAL.getName(), "real_original_string"); put("data", "field1_val,100"); put("metadata_field", "metadata_val"); }}; ProcessorResult<List<byte[]>> results = driver.run(ImmutableList.of(JSONUtils.INSTANCE.toJSONPretty(inputRecord))); Assert.assertFalse(results.failed()); List<byte[]> resultList = results.getResult(); Assert.assertEquals(1, resultList.size()); Map<String, Object> outputRecord = JSONUtils.INSTANCE.load(new String(resultList.get(0)), JSONUtils.MAP_SUPPLIER); Assert.assertEquals("field1_val", outputRecord.get("field1")); Assert.assertEquals(inputRecord.get(Constants.Fields.ORIGINAL.getName()), outputRecord.get(Constants.Fields.ORIGINAL.getName())); Assert.assertEquals(inputRecord.get("metadata_field"), outputRecord.get("metadata_field")); } /**
@Test public void test() throws Exception { ParserDriver driver = new StormParserDriver(sensorType, readSensorConfig(sensorType), readGlobalConfig()); runTest(driver); }