@Override public void process(Event event) { numRecords.mark(); Timer.Context timerContext = mappingTimer.time(); try { Record record = new Record(); for (Entry<String, String> entry : event.getHeaders().entrySet()) { record.put(entry.getKey(), entry.getValue()); } byte[] bytes = event.getBody(); if (bytes != null && bytes.length > 0) { record.put(Fields.ATTACHMENT_BODY, bytes); } try { Notifications.notifyStartSession(morphline); if (!morphline.process(record)) { numFailedRecords.mark(); LOG.warn("Morphline {} failed to process record: {}", morphlineFileAndId, record); } } catch (RuntimeException t) { numExceptionRecords.mark(); morphlineContext.getExceptionHandler().handleException(t, record); } } finally { timerContext.stop(); } }
/** Returns a shallow copy of this record. */ public Record copy() { //return new Record(ArrayListMultimap.create(fields)); // adding fields later causes (slow) rehashing ArrayListMultimap<String,Object> copy = ArrayListMultimap.create(fields.size() + 16, 10); copy.putAll(fields); return new Record(copy); }
/** * Notify a command that a lifecycle event has occurred. * @param command The {@link Command} to be notified. * @param event The {@link LifecycleEvent} to be passed down to the given command. */ private static void notify(Command command, LifecycleEvent event) { Record notification = new Record(); notification.put(LIFE_CYCLE, event); command.notify(notification); }
private Record toRecord(SolrInputDocument doc) { Record record = new Record(); for (Entry<String, SolrInputField> entry : doc.entrySet()) { record.getFields().putAll(entry.getKey(), entry.getValue().getValues()); } return record; }
@Test public void testExternalObject() throws Exception { ArrayListMultimap.create(); ScriptEvaluator script = new ScriptEvaluator( javaImports, "com.google.common.collect.ArrayListMultimap.create(); new org.kitesdk.morphline.api.Record(); return x.copy(); ", Record.class, new String[] { "x" }, new Class[] { Record.class }, "myQuery"); Object result = script.evaluate(new Object[] { new Record() }); assertEquals(result, new Record()); assertTrue(result != new Record()); }
@Test public void testXsltIdentityHelloWorld() throws Exception { morphline = createMorphline("test-morphlines/xslt-helloworld-identity"); InputStream in = new FileInputStream(new File(RESOURCES_DIR + "/test-documents/helloworld.xml")); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, in); processAndVerifySuccess(record, ImmutableMultimap.of("description", "An XSLT Morphline", "welcome", "Hello, World!", "id", "2") ); in.close(); }
@Test public void testIgnoreRecordTooLong() throws Exception { boolean ignoreTooLongRecords = true; int maxCharactersPerRecord = 10; CSVTokenizer tokenizer = new QuotedCSVTokenizer( ',', false, false, new ArrayList<String>(), maxCharactersPerRecord, ignoreTooLongRecords, '"'); assertFalse(tokenizer.tokenizeLine( "\"", new BufferedReader(new StringReader("line tooooooooo long\"")), new Record())); }
@Test public void testReadLineWithMimeTypeWildcard() throws Exception { String threeLines = "first\nsecond\nthird"; byte[] in = threeLines.getBytes("UTF-8"); morphline = createMorphline("test-morphlines/readLineWithMimeTypeWildcard"); // uses ignoreFirstLine : true Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, in); processAndVerifySuccess(record, ImmutableMultimap.of(Fields.MESSAGE, "second"), ImmutableMultimap.of(Fields.MESSAGE, "third") ); }
@Test public void testFindReplaceWithRegex() throws Exception { morphline = createMorphline("test-morphlines/findReplaceWithRegex"); Record record = new Record(); record.put("text", "hello ic world ic"); Record expected = new Record(); expected.put("text", "hello! ic! world! ic!"); processAndVerifySuccess(record, expected); }
private String toString(GenericData.Record avroRecord) { Record record = new Record(); for (Field field : avroRecord.getSchema().getFields()) { record.put(field.name(), avroRecord.get(field.pos())); } return record.toString(); // prints sorted by key for human readability }
@Test public void testReadLineWithMimeTypeMismatch() throws Exception { String threeLines = "first\nsecond\nthird"; byte[] in = threeLines.getBytes("UTF-8"); morphline = createMorphline("test-morphlines/readLineWithMimeTypeMismatch"); // uses ignoreFirstLine : true Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, in); processAndVerifyFailure(record); }
@Test public void testFindReplaceWithGrokWithReplaceFirst() throws Exception { Config override = ConfigFactory.parseString("replaceFirst : true"); morphline = createMorphline("test-morphlines/findReplaceWithGrok", override); Record record = new Record(); record.put("text", "hello ic world ic"); Record expected = new Record(); expected.put("text", "hello! ic world ic"); processAndVerifySuccess(record, expected); }
@Test public void testConvertTimestampWithInputFormatUnixTimeInMillis() throws Exception { morphline = createMorphline("test-morphlines/convertTimestampWithInputFormatUnixTimeInMillis"); Record record = new Record(); record.put("ts1", "0"); record.put("ts1", "1370636123501"); Record expected = new Record(); expected.put("ts1", "1970-01-01T00:00:00.000Z"); expected.put("ts1", "2013-06-07T20:15:23.501Z"); processAndVerifySuccess(record, expected); }
@Test public void testConvertTimestampWithOutputFormatUnixTimeInMillis() throws Exception { morphline = createMorphline("test-morphlines/convertTimestampWithOutputFormatUnixTimeInMillis"); Record record = new Record(); record.put("ts1", "1970-01-01T00:00:00.000Z"); record.put("ts1", "2013-06-07T20:15:23.501Z"); Record expected = new Record(); expected.put("ts1", "0"); expected.put("ts1", "1370636123501"); processAndVerifySuccess(record, expected); }
@Test public void testTranslateFailure() throws Exception { morphline = createMorphline("test-morphlines/translateFailure"); Record record = new Record(); record.replaceValues("level", 999); processAndVerifyFailure(record); }
@Test public void testConvertTimestampBad() throws Exception { morphline = createMorphline("test-morphlines/convertTimestamp"); Record record = new Record(); record.put("ts1", "this is an invalid timestamp"); processAndVerifyFailure(record); }
private void testGenerateUUID(String suffix) throws Exception { morphline = createMorphline("test-morphlines/generateUUID" + suffix); Record record = new Record(); collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); Record actual = collector.getFirstRecord(); assertEquals(1, actual.get("id").size()); String uuid = (String) actual.getFirstValue("id"); assertEquals(36, uuid.length()); }
@Test public void testDetectMimeTypesWithDefaultMimeTypes() throws Exception { morphline = createMorphline("test-morphlines/detectMimeTypesWithDefaultMimeTypes"); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, Files.toByteArray(JPG_FILE)); startSession(); morphline.process(record); assertEquals("image/jpeg", collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_MIME_TYPE)); }
@Test public void testDecodeBase64() throws Exception { morphline = createMorphline("test-morphlines/decodeBase64"); Record record = new Record(); record.put("data", "SGVsbG8gV29ybGQ="); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); byte[] actual = (byte[]) collector.getFirstRecord().getFirstValue("data"); assertArrayEquals("Hello World".getBytes(Charsets.UTF_8), actual); assertSame(record, collector.getFirstRecord()); }