@Override protected boolean doProcess(Record inputRecord, InputStream stream) throws IOException { if (counter++ % 8192 == 0) { blob = new ByteArrayOutputStream(8192); // periodically gc memory from large outlier BLOBs } incrementNumRecords(); blob.reset(); int len; while ((len = stream.read(buffer)) >= 0) { blob.write(buffer, 0, len); } inputRecord.replaceValues(outputFieldName, blob.toByteArray()); // pass record to next command in chain: return getChild().process(inputRecord); }
@Override protected boolean doProcess(Record inputRecord, InputStream stream) throws IOException { if (counter++ % 8192 == 0) { blob = new ByteArrayOutputStream(8192); // periodically gc memory from large outlier BLOBs } incrementNumRecords(); blob.reset(); int len; while ((len = stream.read(buffer)) >= 0) { blob.write(buffer, 0, len); } inputRecord.replaceValues(outputFieldName, blob.toByteArray()); // pass record to next command in chain: return getChild().process(inputRecord); }
@Override protected boolean doProcess(Record record) { if (preserveExisting && record.getFields().containsKey(fieldName)) { // we must preserve the existing timestamp } else { record.replaceValues(fieldName, System.currentTimeMillis()); } // pass record to next command in chain: return super.doProcess(record); }
@Override protected boolean doProcess(Record record) { if (preserveExisting && record.getFields().containsKey(fieldName)) { // we must preserve the existing timestamp } else { record.replaceValues(fieldName, System.currentTimeMillis()); } // pass record to next command in chain: return super.doProcess(record); }
@Override protected boolean doProcess(Record record) { Map<String, Object> map = new HashMap<String, Object>(); for(String key : record.getFields().keySet()){ if(!excludeFields.contains(key)){ map.put(key, record.getFirstValue(key)); } } try { record.replaceValues(Fields.ATTACHMENT_BODY, mapper.writeValueAsString(map)); } catch (JsonProcessingException e) { e.printStackTrace(); } return super.doProcess(record); }
private boolean doProcess2(Record inputRecord, InputStream stream) throws IOException, SAXException { ByteArrayOutputStream out = new ByteArrayOutputStream(16 * 1024); XMLWriter xmlWriter = new XMLWriter(new BufferedWriter(new OutputStreamWriter(out, Charsets.UTF_8))); xmlWriter.setOutputProperty(XMLWriter.ENCODING, "UTF-8"); if (omitXMLDeclaration) { xmlWriter.setOutputProperty(XMLWriter.OMIT_XML_DECLARATION, "yes"); } xmlReader.setContentHandler(xmlWriter); Charset detectedCharset = detectCharset(inputRecord, charset); InputSource source = new InputSource(new BufferedReader(new InputStreamReader(stream, detectedCharset))); xmlReader.parse(source); // push the HTML through tagsoup into the output byte array Record outputRecord = inputRecord.copy(); removeAttachments(outputRecord); outputRecord.replaceValues(Fields.ATTACHMENT_BODY, out.toByteArray()); incrementNumRecords(); // pass record to next command in chain: if (!getChild().process(outputRecord)) { return false; } return true; }
@Override protected boolean doProcess(Record record) { Map<String, Object> map = new HashMap<String, Object>(); for(String key : record.getFields().keySet()){ if(!excludeFields.contains(key)){ map.put(key, record.getFirstValue(key)); } } try { record.replaceValues(Fields.ATTACHMENT_BODY, mapper.writeValueAsString(map)); } catch (JsonProcessingException e) { e.printStackTrace(); } return super.doProcess(record); }
@Override protected boolean doProcess(Record inputRecord, InputStream stream) throws IOException { if (counter++ % 8192 == 0) { clob = new StringBuilder(); // periodically gc memory from large outlier strings } incrementNumRecords(); Charset detectedCharset = detectCharset(inputRecord, charset); Reader reader = new InputStreamReader(stream, detectedCharset); clob.setLength(0); int len; while ((len = reader.read(buffer)) >= 0) { clob.append(buffer, 0, len); } Record outputRecord = inputRecord.copy(); removeAttachments(outputRecord); outputRecord.replaceValues(outputFieldName, clob.toString()); // pass record to next command in chain: return getChild().process(outputRecord); }
@Override protected boolean doProcess(Record inputRecord, InputStream stream) throws IOException { if (counter++ % 8192 == 0) { clob = new StringBuilder(); // periodically gc memory from large outlier strings } incrementNumRecords(); Charset detectedCharset = detectCharset(inputRecord, charset); Reader reader = new InputStreamReader(stream, detectedCharset); clob.setLength(0); int len; while ((len = reader.read(buffer)) >= 0) { clob.append(buffer, 0, len); } Record outputRecord = inputRecord.copy(); removeAttachments(outputRecord); outputRecord.replaceValues(outputFieldName, clob.toString()); // pass record to next command in chain: return getChild().process(outputRecord); }
@Test public void testTranslate() throws Exception { morphline = createMorphline("test-morphlines/translate"); Record record = new Record(); Record expected = new Record(); record.replaceValues("level", "0"); expected.replaceValues("level", "Emergency"); processAndVerifySuccess(record, expected); record.replaceValues("level", 0); expected.replaceValues("level", "Emergency"); processAndVerifySuccess(record, expected); record.replaceValues("level", "1"); expected.replaceValues("level", "Alert"); processAndVerifySuccess(record, expected); record.replaceValues("level", 1); expected.replaceValues("level", "Alert"); processAndVerifySuccess(record, expected); record.replaceValues("level", 999); expected.replaceValues("level", "unknown"); processAndVerifySuccess(record, expected); }
@Override protected boolean doProcess(Record record) { if (preserveExisting && record.getFields().containsKey(fieldName)) { ; // we must preserve the existing id } else { record.replaceValues(fieldName, generateUUID()); } // pass record to next command in chain: return super.doProcess(record); }
@Override protected boolean doProcess(Record record) { if (preserveExisting && record.getFields().containsKey(fieldName)) { ; // we must preserve the existing id } else { record.replaceValues(fieldName, generateUUID()); } // pass record to next command in chain: return super.doProcess(record); }
@Override protected boolean doProcess(Record inputRecord) { Record outputRecord = inputRecord.copy(); // change key Text myTextKey = (Text)inputRecord.getFirstValue(this.keyField); outputRecord.replaceValues(this.keyField, MyWritable.keyStr(myTextKey)); // change value MyWritable myWritableValue = (MyWritable)inputRecord.getFirstValue(this.valueField); outputRecord.replaceValues(this.valueField, MyWritable.valueStr(myWritableValue)); return super.doProcess(outputRecord); } }
@Override protected boolean doProcess(Record record) { for (int i = 0; i < count; i++) { Record next = record.copy(); next.replaceValues(name, i); if (!getChild().process(next)) { return false; } } return true; }
private Record createEvent(InputStream in, ListMultimap<String, Object> headers) { Record record = new Record(); record.getFields().putAll(headers); record.replaceValues(Fields.ATTACHMENT_BODY, in); return record; }
@Override protected boolean doProcess(Record record) { if (preserveExisting && record.getFields().containsKey(fieldName)) { // we must preserve the existing id } else { record.replaceValues(fieldName, seqNum++); } return super.doProcess(record); }
@Test public void testTranslateFailure() throws Exception { morphline = createMorphline("test-morphlines/translateFailure"); Record record = new Record(); record.replaceValues("level", 999); processAndVerifyFailure(record); }
@Test public void testTryRulesFail() throws Exception { morphline = createMorphline("test-morphlines/tryRulesFail"); Record record = new Record(); record.put("first_name", "Nadja"); List<Record> expectedList = Lists.newArrayList(); for (int i = 0; i < 2; i++) { Record expected = record.copy(); expected.put("foo2", "bar2"); expected.replaceValues("iter2", i); expectedList.add(expected); } startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertEquals(expectedList, collector.getRecords()); assertNotSame(record, collector.getRecords().get(0)); }
@Test public void testTryRulesPass() throws Exception { morphline = createMorphline("test-morphlines/tryRulesPass"); Record record = new Record(); record.put("first_name", "Nadja"); List<Record> expectedList = Lists.newArrayList(); for (int i = 0; i < 2; i++) { Record expected = record.copy(); expected.put("foo", "bar"); expected.replaceValues("iter", i); expectedList.add(expected); } startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertEquals(expectedList, collector.getRecords()); assertNotSame(record, collector.getRecords().get(0)); }
@Test public void testTryRulesCatchException() throws Exception { morphline = createMorphline("test-morphlines/tryRulesCatchException"); Record record = new Record(); record.put("first_name", "Nadja"); List<Record> expectedList = Lists.newArrayList(); for (int i = 0; i < 2; i++) { Record expected = record.copy(); expected.put("foo2", "bar2"); expected.replaceValues("iter2", i); expectedList.add(expected); } startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertEquals(expectedList, collector.getRecords()); assertNotSame(record, collector.getRecords().get(0)); }