@Override public Map<String, Object> applyTransformations(SensorParserContext sensorParserContext) { JSONObject sampleJson = new JSONObject(sensorParserContext.getSampleData()); sensorParserContext.getSensorParserConfig().getFieldTransformations().forEach(fieldTransformer -> { fieldTransformer.transformAndUpdate(sampleJson, Context.EMPTY_CONTEXT(), sensorParserContext.getSensorParserConfig().getParserConfig()); } ); return sampleJson; }
/** * Applies Stellar field transformations defined in the sensor parser config. * @param message Message parsed by the MessageParser * @param rawMessage Raw message including metadata * @param sensorParserConfig Sensor parser config */ private void applyFieldTransformations(JSONObject message, RawMessage rawMessage, SensorParserConfig sensorParserConfig) { for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) { if (handler != null) { if (!sensorParserConfig.getMergeMetadata()) { //if we haven't merged metadata, then we need to pass them along as configuration params. handler.transformAndUpdate( message, stellarContext, sensorParserConfig.getParserConfig(), rawMessage.getMetadata() ); } else { handler.transformAndUpdate( message, stellarContext, sensorParserConfig.getParserConfig() ); } } } }
private String transform(String in, String config) throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(config)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ put("in_field", in); put("dummy_field", "dummy"); //this is added to ensure that it looks like something approaching a real message }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); return (String) input.get("out_field"); }
@Test(expected=IllegalStateException.class) public void testStellarBadConfig() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(badConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(); try { handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); } catch(IllegalStateException ex) { Assert.assertTrue(ex.getMessage().contains("URL_TO_HOST")); Assert.assertTrue(ex.getMessage().contains("123")); throw ex; } }
@Test public void testUnconditionalRemove() throws Exception{ SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(removeUnconditionalConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ put("field1", "foo"); }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertFalse(input.containsKey("field1")); }
@Test public void testIntermediateValues() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(intermediateValuesConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); int expected = 3; Assert.assertEquals(expected, input.get("final_value")); Assert.assertFalse(input.containsKey("value1")); Assert.assertFalse(input.containsKey("value2")); }
@Test public void testConfigAll() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(configAll)); JSONObject input = new JSONObject(); input.put("source.type", "test"); for (FieldTransformer handler : c.getFieldTransformations()) { handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); } Assert.assertEquals(2, input.size()); Assert.assertTrue(input.containsKey("new_field")); Assert.assertEquals("test", input.get("new_field")); }
/** * Test the happy path. This ensures that a simple transformation, converting a timestamp in a yyyy-MM-dd HH:mm:ss * format can be converted to the expected UTC MS since Epoch. */ @Test public void testStellar() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(stellarConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ put("timestamp", "2016-01-05 17:02:30"); }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); long expected = 1452013350000L; Assert.assertEquals(expected, input.get("utc_timestamp")); Assert.assertTrue(input.containsKey("timestamp")); }
@Test public void testStellarNumericDomain() throws Exception { /* Despite the domain being weird, URL_TO_HOST should allow it to pass through. However, because it does NOT form a proper domain (no TLD), DOMAIN_REMOVE_SUBDOMAINS returns null indicating that the input is semantically incorrect. */ SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(configNumericDomain)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertTrue(input.containsKey("full_hostname")); Assert.assertEquals("1234567890123456789012345678901234567890123456789012345678901234567890", input.get("full_hostname")); Assert.assertFalse(input.containsKey("domain_without_subdomains")); }
@Test public void testStellarSpecialCharacters() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(stellarConfigEspecial)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ put("timestamp", "2016-01-05 17:02:30"); }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); long expected = 1452013350000L; Assert.assertEquals(expected, input.get("utc_timestamp")); Assert.assertTrue(input.containsKey("timestamp")); Assert.assertTrue(input.containsKey("newStellarField")); }
@Test public void renameMissingField() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(renameMissingField)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ for(int i = 2;i <= 10;++i) { put("old_field" + i, "f" + i); } }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertFalse(input.containsKey("new_field1")); for(int i = 2;i <= 10;++i) { Assert.assertEquals("f" + i, input.get("old_field" + i)); } Assert.assertEquals(9, input.size()); } }
/** * Ensures that if we try to transform with a field which does not exist, it does not * 1. throw an exception * 2. do any transformation. */ @Test public void testStellar_negative() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(stellarConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); //no input fields => no transformation JSONObject input = new JSONObject(new HashMap<String, Object>() {{ }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertFalse(input.containsKey("utc_timestamp")); Assert.assertTrue(input.isEmpty()); }
@Test public void smokeTest() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(smoketestConfig)); FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() {{ for(int i = 1;i <= 10;++i) { put("old_field" + i, "f" + i); } }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertEquals("f1", input.get("new_field1")); Assert.assertEquals("f2", input.get("new_field2")); for(int i = 3;i <= 10;++i) { Assert.assertEquals("f" + i, input.get("old_field" + i)); } Assert.assertFalse(input.containsKey("old_field1")); Assert.assertFalse(input.containsKey("old_field2")); Assert.assertEquals(10, input.size()); }
@Test public void testSingleFieldReturned() throws Exception { SensorParserConfig sensorConfig = SensorParserConfig.fromBytes(Bytes.toBytes(selectSingleFieldConfig)); FieldTransformer handler = Iterables.getFirst(sensorConfig.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() { { put("field1", "foo"); put("field2", "bar"); } }); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertTrue(input.containsKey("field1")); Assert.assertFalse(input.containsKey("field2")); Assert.assertEquals(1, input.size()); }
@Test public void testStellarRename() throws Exception { SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(configRename)); { JSONObject input = new JSONObject(); input.put("old_field", "val"); input.put("old_field2", "val2"); for (FieldTransformer handler : c.getFieldTransformations()) { handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); } Assert.assertEquals(2, input.size()); Assert.assertTrue(input.containsKey("new_field")); Assert.assertEquals("val", input.get("new_field")); Assert.assertEquals("val2", input.get("new_field2")); Assert.assertTrue(!input.containsKey("old_field")); Assert.assertTrue(!input.containsKey("old_field2")); } { JSONObject input = new JSONObject(); input.put("old_field", "val"); for (FieldTransformer handler : c.getFieldTransformations()) { handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); } Assert.assertEquals(1, input.size()); Assert.assertTrue(input.containsKey("new_field")); Assert.assertEquals("val", input.get("new_field")); } }
put("field1", "foo"); }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); put("field2", "bar"); }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertFalse(input.containsKey("field1")); Assert.assertTrue(input.containsKey("field2"));
@Test public void testMulitpleFieldReturned() throws Exception { SensorParserConfig sensorConfig = SensorParserConfig.fromBytes(Bytes.toBytes(selectMultiFieldConfig)); FieldTransformer handler = Iterables.getFirst(sensorConfig.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() { { put("field1", "foo"); put("field2", "bar"); put("field3", "bar2"); } }); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertTrue(input.containsKey("field1")); Assert.assertTrue(input.containsKey("field2")); Assert.assertFalse(input.containsKey("field3")); Assert.assertEquals(2, input.size()); }
handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); long expected = 1452013350000L; Assert.assertEquals(expected, input.get("utc_timestamp")); put("dc", "london"); }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT(), c.getParserConfig()); long expected = 1452013350000L; Assert.assertEquals(expected, input.get("utc_timestamp")); put("url", "https://caseystella.com/blog"); }}); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT(), c.getParserConfig()); long expected = 1452013350000L; Assert.assertEquals(expected, input.get("utc_timestamp"));
@Test public void testPreserveSystemFields() throws Exception { SensorParserConfig sensorConfig = SensorParserConfig.fromBytes(Bytes.toBytes(selectSingleFieldConfig)); FieldTransformer handler = Iterables.getFirst(sensorConfig.getFieldTransformations(), null); JSONObject input = new JSONObject(new HashMap<String, Object>() { { put("timestamp", 12345); put("original_string", "foo,bar"); put("source.type", "test"); put("field1", "foo"); put("field2", "bar"); } }); handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); Assert.assertTrue(input.containsKey("timestamp")); Assert.assertTrue(input.containsKey("original_string")); Assert.assertTrue(input.containsKey("source.type")); Assert.assertTrue(input.containsKey("field1")); Assert.assertFalse(input.containsKey("field2")); Assert.assertEquals(4, input.size()); }