@Override public void process(Event event) { numRecords.mark(); Timer.Context timerContext = mappingTimer.time(); try { Record record = new Record(); for (Entry<String, String> entry : event.getHeaders().entrySet()) { record.put(entry.getKey(), entry.getValue()); } byte[] bytes = event.getBody(); if (bytes != null && bytes.length > 0) { record.put(Fields.ATTACHMENT_BODY, bytes); } try { Notifications.notifyStartSession(morphline); if (!morphline.process(record)) { numFailedRecords.mark(); LOG.warn("Morphline {} failed to process record: {}", morphlineFileAndId, record); } } catch (RuntimeException t) { numExceptionRecords.mark(); morphlineContext.getExceptionHandler().handleException(t, record); } } finally { timerContext.stop(); } }
private void addValue(Record record, Object value) { if (value != null) { record.put(outputFieldName, value); } } }
private void tryAddLong(Record doc, String solr_field, JsonNode node) { if (node == null) return; Long val = node.asLong(); if (val == null) { return; } doc.put(solr_field, val); }
private void extractColumns(Record record, Iterable<String> columns) { Iterator<String> iter = columns.iterator(); for (int i = 0; i < outputFieldNames.size() && iter.hasNext(); i++) { String columnValue = iter.next(); String columnName = outputFieldNames.get(i); if (columnName.length() > 0) { // empty column name indicates omit this field on output if (columnValue.length() > 0 || addEmptyStrings) { record.put(columnName, columnValue); } } } }
@Test public void testConvertTimestampWithOutputFormatUnixTimeInSeconds() throws Exception { morphline = createMorphline("test-morphlines/convertTimestampWithOutputFormatUnixTimeInSeconds"); Record record = new Record(); record.put("ts1", "1970-01-01T00:00:00.000Z"); record.put("ts1", "2013-06-07T20:15:23.501Z"); Record expected = new Record(); expected.put("ts1", "0"); expected.put("ts1", "1370636123"); processAndVerifySuccess(record, expected); }
@Test public void testSplitWithOutputFields() throws Exception { morphline = createMorphline("test-morphlines/splitWithOutputFields"); Record record = new Record(); String msg = " _a ,_b_ , ,c__,d"; record.put(Fields.MESSAGE, msg); Record expected = new Record(); expected.put(Fields.MESSAGE, msg); expected.put("col0", "_a"); expected.put("col2", "c__"); processAndVerifySuccess(record, expected); }
@Test public void testJavaHelloWorld() throws Exception { morphline = createMorphline("test-morphlines/javaHelloWorld"); Record record = new Record(); record.put("tags", "hello"); Record expected = new Record(); expected.put("tags", "hello"); expected.put("tags", "world"); processAndVerifySuccess(record, expected); }
@Test public void testXQueryTweetUsers() throws Exception { morphline = createMorphline("test-morphlines/xquery-tweet-users"); InputStream in = new FileInputStream(new File(RESOURCES_DIR + "/test-documents/sample-statuses-20120906-141433.xml")); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, in); processAndVerifySuccess(record, ImmutableMultimap.of("followers_count", "111", "id", "11111112", "screen_name", "fake_user1", "greeting", "hello world", "annotation", "An XSLT Morphline"), ImmutableMultimap.of("followers_count", "222", "id", "222223", "screen_name", "fake_user2", "greeting", "hello world", "annotation", "An XSLT Morphline") ); in.close(); }
@Test public void testXQueryAtomicValues() throws Exception { morphline = createMorphline("test-morphlines/xquery-atomic-values"); InputStream in = new FileInputStream(new File(RESOURCES_DIR + "/test-documents/sample-statuses-20120906-141433.xml")); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, in); processAndVerifySuccess(record); in.close(); }
@Test public void testReadLineWithMimeType() throws Exception { String threeLines = "first\nsecond\nthird"; byte[] in = threeLines.getBytes("UTF-8"); morphline = createMorphline("test-morphlines/readLineWithMimeType"); // uses ignoreFirstLine : true Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, in); processAndVerifySuccess(record, ImmutableMultimap.of(Fields.MESSAGE, "second"), ImmutableMultimap.of(Fields.MESSAGE, "third") ); }
private void testAddLocalHostInternal(String name) throws Exception { Record record = new Record(); Record expected = new Record(); expected.put("myhost", name); processAndVerifySuccess(record, expected); // test that preserveExisting = true preserves the existing value record = new Record(); record.put("myhost", "myname"); expected = record.copy(); processAndVerifySuccess(record, expected); }
@Test public void testFindReplace() throws Exception { Config override = ConfigFactory.parseString("replaceFirst : false"); morphline = createMorphline("test-morphlines/findReplace", override); Record record = new Record(); record.put("text", "hello ic world ic"); Record expected = new Record(); expected.put("text", "hello I see world I see"); processAndVerifySuccess(record, expected); }
@Test public void testFindReplaceWithGrok() throws Exception { Config override = ConfigFactory.parseString("replaceFirst : false"); morphline = createMorphline("test-morphlines/findReplaceWithGrok", override); Record record = new Record(); record.put("text", "hello ic world ic"); Record expected = new Record(); expected.put("text", "hello! ic! world! ic!"); processAndVerifySuccess(record, expected); }
@Test public void testParseInclude() throws Exception { morphline = createMorphline("test-morphlines/parseInclude"); Record input = new Record(); Record expected = new Record(); expected.put("result", "bar"); processAndVerifySuccess(input, expected); }
@Test public void testParseIncludeConstants() throws Exception { morphline = createMorphline("test-morphlines/parseIncludeConstants"); Record input = new Record(); Record expected = new Record(); expected.put("myField", "foo"); processAndVerifySuccess(input, expected); }
@Test public void testRemoveAllFields() throws Exception { morphline = createMorphline("test-morphlines/removeAllFields"); Record record = new Record(); record.put("foo", "data"); processAndVerifySuccess(record, new Record()); }
@Test public void testGrokWithEscaping() throws Exception { morphline = createMorphline("test-morphlines/grokWithEscaping"); Record record = new Record(); record.put(Fields.MESSAGE, "{SystemSourceId}=foo"); assertTrue(morphline.process(record)); assertSame(record, collector.getFirstRecord()); record = new Record(); record.put(Fields.MESSAGE, "SystemSourceId}=foo"); // missing opening brace assertFalse(morphline.process(record)); }