private void processAndVerifySuccess(Record input, Record expected, boolean isSame) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); assertEquals(expected, collector.getFirstRecord()); if (isSame) { assertSame(input, collector.getFirstRecord()); } else { assertNotSame(input, collector.getFirstRecord()); } }
private void processAndVerifyFailure(Record input) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertFalse(morphline.process(input)); assertEquals(0, collector.getRecords().size()); }
@Before public void setUp() throws Exception { collector = new Collector(); }
private String detect(Record event, boolean includeMetaData, boolean excludeParameters) throws IOException { List key = Arrays.asList(includeMetaData, excludeParameters); Command cachedMorphline = morphlineCache.get(key); if (cachedMorphline == null) { // avoid recompiling time and again (performance) Config override = ConfigFactory.parseString("INCLUDE_META_DATA : " + includeMetaData + "\nEXCLUDE_PARAMETERS : " + excludeParameters); cachedMorphline = createMorphline("test-morphlines/detectMimeTypesWithDefaultMimeTypesAndFile", override); morphlineCache.put(key, cachedMorphline); } collector.reset(); assertTrue(cachedMorphline.process(event)); String mimeType = (String) collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_MIME_TYPE); return mimeType; }
@Test public void testNotWithTrue() throws Exception { morphline = createMorphline("test-morphlines/notWithTrue"); Record record = createBasicRecord(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertFalse(morphline.process(record)); assertEquals(Lists.newArrayList(), collector.getRecords()); }
@Test public void testExtractJsonPathsFlattened() throws Exception { morphline = createMorphline("test-morphlines/extractJsonPathsFlattened"); File file = new File(RESOURCES_DIR + "/test-documents/arrays.json"); InputStream in = new FileInputStream(file); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, in); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertEquals(1, collector.getRecords().size()); List expected = Arrays.asList(1, 2, 3, 4, 5, 10, 20, 100, 200); assertEquals(1, collector.getRecords().size()); assertEquals(expected, collector.getFirstRecord().get("/price")); assertEquals(expected, collector.getFirstRecord().get("/price/[]")); assertEquals(Arrays.asList(), collector.getFirstRecord().get("/unknownField")); in.close(); }
record.put(Fields.ID, id); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); Record expected = new Record(); expected.put("word", "world"); expected.put("word", "foo"); assertEquals(expected, collector.getFirstRecord()); if (inplace) { assertSame(record, collector.getFirstRecord()); } else { assertNotSame(record, collector.getFirstRecord()); collector.reset(); record = new Record(); record.put(Fields.MESSAGE, ""); record.put(Fields.ID, id); startSession(); assertEquals(1, collector.getNumStartEvents()); assertFalse(morphline.process(record)); assertEquals(Lists.newArrayList(), collector.getRecords());
@Test @Ignore public void benchmarkJson() throws Exception { String morphlineConfigFile = "test-morphlines/readJson"; long durationSecs = 10; //File file = new File(RESOURCES_DIR + "/test-documents/stream.json"); File file = new File(RESOURCES_DIR + "/test-documents/sample-statuses-20120906-141433.json"); System.out.println("Now benchmarking " + morphlineConfigFile + " ..."); morphline = createMorphline(morphlineConfigFile); byte[] bytes = Files.toByteArray(file); long start = System.currentTimeMillis(); long duration = durationSecs * 1000; int iters = 0; while (System.currentTimeMillis() < start + duration) { Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, bytes); collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); iters++; } float secs = (System.currentTimeMillis() - start) / 1000.0f; System.out.println("Results: iters=" + iters + ", took[secs]=" + secs + ", iters/secs=" + (iters/secs)); }
assertEquals(1, collector.getRecords().size()); assertEquals(Arrays.asList("nadja"), collector.getFirstRecord().get("/mapField/foo/label")); assertEquals(Arrays.asList(), collector.getFirstRecord().get("/unknownField")); assertEquals(1, collector.getRecords().size()); assertEquals(Arrays.asList("nadja"), collector.getFirstRecord().get("/mapField/foo/label")); assertEquals(Arrays.asList(), collector.getFirstRecord().get("/unknownField")); Record first = collector.getFirstRecord(); assertEquals(Arrays.asList("nadja"), first.get("/mapField/foo/label")); AbstractParser.removeAttachments(first); )) ); collector.reset(); assertFalse(morphline.process(jdoc0)); )) ); collector.reset(); assertFalse(morphline.process(jdoc0)); )) ); collector.reset(); assertTrue(morphline.process(jdoc0)); GenericData.Record result = (GenericData.Record) collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_BODY); GenericData.Record result2 = (GenericData.Record) ((Map)result.get("mapField")).get(utf8("foo")); assertEquals("nadja", result2.get("label"));
@Test public void testBasic() throws IOException { String msg = "hello world"; // setup: copy a file to HDFS to prepare inputFile Path inputFile = fileSystem.makeQualified(new Path(testDirectory, "foo.txt.gz")); OutputStream out = fileSystem.create(inputFile); out = new GZIPOutputStream(out); IOUtils.copyBytes(new ByteArrayInputStream(msg.getBytes(Charsets.UTF_8)), out, fileSystem.getConf()); out.flush(); out.close(); Assert.assertTrue(fileSystem.exists(inputFile)); Command morphline = createMorphline("test-morphlines/openHdfsFile"); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, inputFile.toString()); Assert.assertTrue(morphline.process(record)); Record expected = new Record(); expected.put(Fields.MESSAGE, msg); Assert.assertEquals(expected, collector.getFirstRecord()); }
@Test public void testNotWithFalseWithContinuation() throws Exception { morphline = createMorphline("test-morphlines/notWithFalseAndContinuation"); Record record = createBasicRecord(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertEquals(record, collector.getFirstRecord()); assertSame(record, collector.getFirstRecord()); assertEquals("touched", collector.getFirstRecord().getFirstValue("state")); }
@Test public void testExtractAvroPathsArrayInUnion() throws Exception { List<String> items = Arrays.asList("a", "b", "c"); ArrayInUnionTestRecord avroRecord = new ArrayInUnionTestRecord(items, items); morphline = createMorphline("test-morphlines/extractAvroPathsArrayInUnion"); deleteAllDocuments(); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, avroRecord); startSession(); assertTrue(morphline.process(record)); assertEquals(1, collector.getRecords().size()); assertEquals(Arrays.asList(items), collector.getFirstRecord().get("/items[]")); assertEquals(Arrays.asList(items), collector.getFirstRecord().get("/itemsInUnion[]")); }
private int queryResultSetSize(String query) { return collector.getRecords().size(); }
morphline = createMorphline("test-morphlines/readAvroContainer"); for (int run = 0; run < 10; run++) { collector.reset(); int version = run % 2; version = (version + reverse) % 2; // reverse direction with reverse == 1: 0 -> 1 as well as 1 -> 0 assertEquals(numRecords, collector.getRecords().size()); Record record = collector.getRecords().get(i); GenericData.Record avroRecord = (GenericData.Record)record.getFirstValue(Fields.ATTACHMENT_BODY); assertEquals(expectedUids[i], avroRecord.get("sc_uid").toString());
protected void deleteAllDocuments() { collector.reset(); }
@Test public void testLoadSolrWithChildDocuments() throws Exception { morphline = createMorphline("test-morphlines" + File.separator + "loadSolrWithChildDocuments"); Record record = new Record(); record.put(Fields.ID, "id0"); startSession(); Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record)); assertEquals(1, collector.getNumStartEvents()); Notifications.notifyCommitTransaction(morphline); // This parent block join returns the parent records for records // where the child documents contain "bar" in the id field. SolrDocumentList docs = query("{!parent which='content_type:parent'}id:bar").getResults(); assertEquals(1, docs.size()); assertEquals("id0", docs.get(0).getFirstValue(Fields.ID)); docs = query("*:*").getResults(); assertEquals(3, docs.size()); }
@Test public void testTryRulesCatchException() throws Exception { morphline = createMorphline("test-morphlines/tryRulesCatchException"); Record record = new Record(); record.put("first_name", "Nadja"); List<Record> expectedList = Lists.newArrayList(); for (int i = 0; i < 2; i++) { Record expected = record.copy(); expected.put("foo2", "bar2"); expected.replaceValues("iter2", i); expectedList.add(expected); } startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertEquals(expectedList, collector.getRecords()); assertNotSame(record, collector.getRecords().get(0)); }
@Test public void testExtractJsonPaths() throws Exception { morphline = createMorphline("test-morphlines/extractJsonPaths"); File file = new File(RESOURCES_DIR + "/test-documents/arrays.json"); InputStream in = new FileInputStream(file); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, in); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertEquals(1, collector.getRecords().size()); JsonNode rootNode = (JsonNode) new ObjectMapper().reader(JsonNode.class).readValues(file).next(); assertTrue(rootNode.get("price").isArray()); List<JsonNode> expected = Arrays.asList(rootNode.get("price")); assertEquals(1, collector.getRecords().size()); assertEquals(expected, collector.getFirstRecord().get("/price")); assertEquals(expected, collector.getFirstRecord().get("/price/[]")); assertEquals(Arrays.asList(), collector.getFirstRecord().get("/unknownField")); in.close(); }
String id = "myid"; record.put(Fields.ID, id); collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); Record expected = new Record(); expected.put("syslog_pid", "607"); expected.put("syslog_message", "Server listening on 0.0.0.0 port 22."); assertEquals(expected, collector.getFirstRecord()); if (inplace) { assertSame(record, collector.getFirstRecord()); } else { assertNotSame(record, collector.getFirstRecord()); collector.reset(); record = new Record(); record.put(Fields.MESSAGE, "foo" + msg); startSession(); assertEquals(1, collector.getNumStartEvents()); assertFalse(morphline.process(record)); assertEquals(Lists.newArrayList(), collector.getRecords()); collector.reset(); record = new Record(); record.put(Fields.MESSAGE, msg); record.put(Fields.ID, id);
event.getFields().put(Fields.ATTACHMENT_NAME, f.getName()); event.getFields().put(Fields.BASE_ID, f.getName()); collector.reset(); load(event); Integer count = expectedRecords.get(file); Map<String, Object> expectedContents = expectedRecordContents.get(file); if (expectedContents != null) { Record actual = collector.getFirstRecord(); for (Map.Entry<String, Object> entry : expectedContents.entrySet()) { if (entry.getValue() == NON_EMPTY_FIELD) {