protected void deleteAllDocuments() { collector.reset(); }
public Collector() { reset(); }
private void deleteAllDocuments() throws SolrServerException, IOException { collector.reset(); SolrClient s = solrClient; s.deleteByQuery("*:*"); // delete everything! s.commit(); }
private String detect(Record event, boolean includeMetaData, boolean excludeParameters) throws IOException { List key = Arrays.asList(includeMetaData, excludeParameters); Command cachedMorphline = morphlineCache.get(key); if (cachedMorphline == null) { // avoid recompiling time and again (performance) Config override = ConfigFactory.parseString("INCLUDE_META_DATA : " + includeMetaData + "\nEXCLUDE_PARAMETERS : " + excludeParameters); cachedMorphline = createMorphline("test-morphlines/detectMimeTypesWithDefaultMimeTypesAndFile", override); morphlineCache.put(key, cachedMorphline); } collector.reset(); assertTrue(cachedMorphline.process(event)); String mimeType = (String) collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_MIME_TYPE); return mimeType; }
private void processAndVerifySuccess(Record input, Multimap... expectedMaps) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); Iterator<Record> iter = collector.getRecords().iterator(); for (Multimap expected : expectedMaps) { assertTrue(iter.hasNext()); Record record = iter.next(); assertEquals(expected, record.getFields()); } assertFalse(iter.hasNext()); }
private void processAndVerifySuccess(Record input, Multimap... expectedMaps) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); Iterator<Record> iter = collector.getRecords().iterator(); int i = 0; for (Multimap expected : expectedMaps) { //System.out.println("iter" + i); assertTrue(iter.hasNext()); assertEquals(expected, iter.next().getFields()); i++; } assertFalse(iter.hasNext()); }
private void processAndVerifySuccess(Record input, Multimap... expectedMaps) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); Iterator<Record> iter = collector.getRecords().iterator(); int i = 0; for (Multimap expected : expectedMaps) { //System.out.println("iter" + i); assertTrue(iter.hasNext()); assertEquals(expected, iter.next().getFields()); i++; } assertFalse(iter.hasNext()); }
private void processAndVerifySuccess(Command myMorphline, Collector myCollector, Record input, Multimap... expectedMaps) { myCollector.reset(); Notifications.notifyStartSession(myMorphline); assertEquals(1, myCollector.getNumStartEvents()); assertTrue(myMorphline.process(input)); Iterator<Record> iter = myCollector.getRecords().iterator(); int i = 0; for (Multimap expected : expectedMaps) { //System.out.println("iter" + i); assertTrue(iter.hasNext()); assertEquals(expected, iter.next().getFields()); i++; } assertFalse(iter.hasNext()); }
private void processAndVerifyFailure(Record input) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertFalse(morphline.process(input)); assertEquals(0, collector.getRecords().size()); }
private void testGenerateUUID(String suffix) throws Exception { morphline = createMorphline("test-morphlines/generateUUID" + suffix); Record record = new Record(); collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); Record actual = collector.getFirstRecord(); assertEquals(1, actual.get("id").size()); String uuid = (String) actual.getFirstValue("id"); assertEquals(36, uuid.length()); }
private void processAndVerifySuccess(Record input, Record expected, boolean isSame) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); assertEquals(expected, collector.getFirstRecord()); if (isSame) { assertSame(input, collector.getFirstRecord()); } else { assertNotSame(input, collector.getFirstRecord()); } }
private void processAndVerifySuccess(Record input, Record expected, boolean isSame) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); assertEquals(expected, collector.getFirstRecord()); if (isSame) { assertSame(input, collector.getFirstRecord()); } else { assertNotSame(input, collector.getFirstRecord()); } }
private void processAndVerifySuccess(Record input, Record expected, boolean isSame) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); assertEquals(expected, collector.getFirstRecord()); if (isSame) { assertSame(input, collector.getFirstRecord()); } else { assertNotSame(input, collector.getFirstRecord()); } }
@Test public void testReadClobWithDestination() throws IOException { morphline = createMorphline("test-morphlines/readClobDestField"); for (int i = 0; i < 3; i++) { Record record = new Record(); String msg = "foo"; record.put(Fields.ATTACHMENT_BODY, msg.getBytes("UTF-8")); collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); Record expected = new Record(); expected.put("myAwesomeDestination", msg); assertEquals(expected, collector.getFirstRecord()); assertNotSame(record, collector.getFirstRecord()); } }
@Test public void testReadBlob() throws Exception { morphline = createMorphline("test-morphlines/readBlob"); for (int i = 0; i < 3; i++) { Record record = new Record(); byte[] bytes = "foo".getBytes("UTF-8"); record.put(Fields.ATTACHMENT_BODY, bytes); collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertSame(record, collector.getFirstRecord()); assertNotSame(bytes, record.getFirstValue(Fields.ATTACHMENT_BODY)); assertArrayEquals(bytes, (byte[])record.getFirstValue(Fields.ATTACHMENT_BODY)); } }
@Test public void testReadClob() throws Exception { morphline = createMorphline("test-morphlines/readClob"); for (int i = 0; i < 3; i++) { Record record = new Record(); String msg = "foo"; record.put(Fields.ATTACHMENT_BODY, msg.getBytes("UTF-8")); collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); Record expected = new Record(); expected.put(Fields.MESSAGE, msg); assertEquals(expected, collector.getFirstRecord()); assertNotSame(record, collector.getFirstRecord()); } }
@Test public void testToByteArray() throws Exception { morphline = createMorphline("test-morphlines/toByteArray"); Record record = new Record(); record.put("first_name", "Nadja"); collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertEquals(1, collector.getFirstRecord().getFields().size()); byte[] expected = "Nadja".getBytes("UTF-8"); assertArrayEquals(expected, (byte[]) collector.getFirstRecord().getFirstValue("first_name")); assertSame(record, collector.getFirstRecord()); }
@Test public void testReadBlobWithDestination() throws IOException { morphline = createMorphline("test-morphlines/readBlobWithOutputField"); for (int i = 0; i < 3; i++) { Record record = new Record(); byte[] bytes = "foo".getBytes("UTF-8"); record.put(Fields.ATTACHMENT_BODY, bytes); collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertSame(record, collector.getFirstRecord()); assertSame(bytes, record.getFirstValue(Fields.ATTACHMENT_BODY)); assertNotSame(bytes, record.getFirstValue("myAwesomeDestination")); assertArrayEquals(bytes, (byte[])record.getFirstValue("myAwesomeDestination")); } }
@Test public void testTokenizeText() throws Exception { morphline = createMorphline("test-morphlines" + File.separator + "tokenizeText"); for (int i = 0; i < 3; i++) { Record record = new Record(); record.put(Fields.MESSAGE, "Hello World!"); record.put(Fields.MESSAGE, "\nFoo@Bar.com #%()123"); Record expected = record.copy(); expected.getFields().putAll("tokens", Arrays.asList("hello", "world", "foo", "bar.com", "123")); collector.reset(); startSession(); Notifications.notifyBeginTransaction(morphline); assertTrue(morphline.process(record)); assertEquals(1, collector.getNumStartEvents()); Notifications.notifyCommitTransaction(morphline); assertEquals(expected, collector.getFirstRecord()); } }
private void testDetectMimeTypesInternal(String configFile) throws Exception { // verify that Avro is classified as Avro morphline = createMorphline(configFile); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, Files.toByteArray(AVRO_FILE)); startSession(); morphline.process(record); assertEquals(AVRO_MIME_TYPE, collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_MIME_TYPE)); // verify that JPG isnt' classified as JPG because this morphline uses includeDefaultMimeTypes : false collector.reset(); record = new Record(); record.put(Fields.ATTACHMENT_BODY, Files.toByteArray(JPG_FILE)); startSession(); morphline.process(record); assertEquals("application/octet-stream", collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_MIME_TYPE)); }