@Override public void process(Event event) { numRecords.mark(); Timer.Context timerContext = mappingTimer.time(); try { Record record = new Record(); for (Entry<String, String> entry : event.getHeaders().entrySet()) { record.put(entry.getKey(), entry.getValue()); } byte[] bytes = event.getBody(); if (bytes != null && bytes.length > 0) { record.put(Fields.ATTACHMENT_BODY, bytes); } try { Notifications.notifyStartSession(morphline); if (!morphline.process(record)) { numFailedRecords.mark(); LOG.warn("Morphline {} failed to process record: {}", morphlineFileAndId, record); } } catch (RuntimeException t) { numExceptionRecords.mark(); morphlineContext.getExceptionHandler().handleException(t, record); } } finally { timerContext.stop(); } }
String morphlineId = context.getString(MORPHLINE_ID_PARAM); if (morphlineFile == null || morphlineFile.trim().length() == 0) { throw new MorphlineCompilationException("Missing parameter: " + MORPHLINE_FILE_PARAM, null); context.getString(FaultTolerance.RECOVERABLE_EXCEPTION_CLASSES)); morphlineContext = new MorphlineContext.Builder() .setExceptionHandler(faultTolerance) .setMetricRegistry(SharedMetricRegistries.getOrCreate(morphlineFileAndId)) .build(); new File(morphlineFile), morphlineId, morphlineContext, finalChild, override); this.mappingTimer = morphlineContext.getMetricRegistry().timer( MetricRegistry.name("morphline.app", Metrics.ELAPSED_TIME)); this.numRecords = morphlineContext.getMetricRegistry().meter( MetricRegistry.name("morphline.app", Metrics.NUM_RECORDS)); this.numFailedRecords = morphlineContext.getMetricRegistry().meter( MetricRegistry.name("morphline.app", "numFailedRecords")); this.numExceptionRecords = morphlineContext.getMetricRegistry().meter( MetricRegistry.name("morphline.app", "numExceptionRecords"));
private Event toEvent(Record record) { Map<String, String> headers = new HashMap(); Map<String, Collection<Object>> recordMap = record.getFields().asMap(); byte[] body = null; for (Map.Entry<String, Collection<Object>> entry : recordMap.entrySet()) {
@Test public void testNotWithFalseWithContinuation() throws Exception { morphline = createMorphline("test-morphlines/notWithFalseAndContinuation"); Record record = createBasicRecord(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertEquals(record, collector.getFirstRecord()); assertSame(record, collector.getFirstRecord()); assertEquals("touched", collector.getFirstRecord().getFirstValue("state")); }
@Test public void testGrokWithEscaping() throws Exception { morphline = createMorphline("test-morphlines/grokWithEscaping"); Record record = new Record(); record.put(Fields.MESSAGE, "{SystemSourceId}=foo"); assertTrue(morphline.process(record)); assertSame(record, collector.getFirstRecord()); record = new Record(); record.put(Fields.MESSAGE, "SystemSourceId}=foo"); // missing opening brace assertFalse(morphline.process(record)); }
@Test public void testGrokWithMultipleGroupsWithSameName() throws Exception { morphline = createMorphline("test-morphlines/grokWithMultipleGroupsWithSameName"); Record record = new Record(); record.put(Fields.MESSAGE, "123 456"); assertTrue(morphline.process(record)); Record expected = new Record(); expected.put(Fields.MESSAGE, "123 456"); expected.put("syslog_pri", "123"); expected.put("syslog_pri", "456"); assertEquals(expected, collector.getFirstRecord()); assertNotSame(record, collector.getFirstRecord()); }
/** * Notify a command that a lifecycle event has occurred. * @param command The {@link Command} to be notified. * @param event The {@link LifecycleEvent} to be passed down to the given command. */ private static void notify(Command command, LifecycleEvent event) { Record notification = new Record(); notification.put(LIFE_CYCLE, event); command.notify(notification); }
@Test public void testHeadUnlimited() throws Exception { morphline = createMorphline("test-morphlines/headUnlimited"); int size = 5; for (int i = 0; i < size; i++) { assertTrue(morphline.process(new Record())); } assertEquals(size, collector.getRecords().size()); }
private void testAddLocalHostInternal(String name) throws Exception { Record record = new Record(); Record expected = new Record(); expected.put("myhost", name); processAndVerifySuccess(record, expected); // test that preserveExisting = true preserves the existing value record = new Record(); record.put("myhost", "myname"); expected = record.copy(); processAndVerifySuccess(record, expected); }
@Test public void testIfThenElseWithElseEmpty() throws Exception { morphline = createMorphline("test-morphlines/ifThenElseWithElseEmpty"); Record record = createBasicRecord(); processAndVerifySuccess(record, record); assertEquals("init1", collector.getFirstRecord().getFirstValue("state")); }
@Override protected boolean doProcess(Record record) { for (int i = 0; i < count; i++) { Record next = record.copy(); next.replaceValues(name, i); if (!getChild().process(next)) { return false; } } return true; }
private void processAndVerifySuccess(Record input, Record expected, boolean isSame) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); assertEquals(expected, collector.getFirstRecord()); if (isSame) { assertSame(input, collector.getFirstRecord()); } else { assertNotSame(input, collector.getFirstRecord()); } }
private void processAndVerifyFailure(Record input) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertFalse(morphline.process(input)); assertEquals(0, collector.getRecords().size()); }
@Test public void testNotWithFalse() throws Exception { morphline = createMorphline("test-morphlines/notWithFalse"); Record record = createBasicRecord(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertEquals(record, collector.getFirstRecord()); assertSame(record, collector.getFirstRecord()); }
/** * Notify a command that a lifecycle event has occurred. * @param command The {@link Command} to be notified. * @param event The {@link LifecycleEvent} to be passed down to the given command. */ private static void notify(Command command, LifecycleEvent event) { Record notification = new Record(); notification.put(LIFE_CYCLE, event); command.notify(notification); }
@Test public void testHead() throws Exception { morphline = createMorphline("test-morphlines/head"); int size = 5; for (int i = 0; i < size; i++) { assertTrue(morphline.process(new Record())); } assertEquals(3, collector.getRecords().size()); }
@Test public void testIfThenElseWithThenEmpty() throws Exception { morphline = createMorphline("test-morphlines/ifThenElseWithThenEmpty"); Record record = createBasicRecord(); processAndVerifySuccess(record, record); assertEquals("init1", collector.getFirstRecord().getFirstValue("state")); }
@Test public void testIfThenElseWithThen() throws Exception { morphline = createMorphline("test-morphlines/ifThenElseWithThen"); Record record = createBasicRecord(); processAndVerifySuccess(record, record); assertEquals("then1", collector.getFirstRecord().getFirstValue("state")); }
@Test public void testIfThenElseWithElse() throws Exception { morphline = createMorphline("test-morphlines/ifThenElseWithElse"); Record record = createBasicRecord(); processAndVerifySuccess(record, record); assertEquals("else1", collector.getFirstRecord().getFirstValue("state")); }
@Test public void testIfThenElseWithThenAndNot() throws Exception { morphline = createMorphline("test-morphlines/ifThenElseWithThenAndNot"); Record record = createBasicRecord(); processAndVerifySuccess(record, record); assertEquals("then1", collector.getFirstRecord().getFirstValue("state")); }