private Path getAttachmentPath(Record record) { // We have more meaningful RCFile error messages if we have an attachment name String attachmentName = (String) record.getFirstValue(Fields.ATTACHMENT_NAME); if (attachmentName == null) { attachmentName = "UNKNOWN"; } return new Path(STREAM_PROTOCOL + attachmentName); }
private Path getAttachmentPath(Record record) { // We have more meaningful RCFile error messages if we have an attachment name String attachmentName = (String) record.getFirstValue(Fields.ATTACHMENT_NAME); if (attachmentName == null) { attachmentName = "UNKNOWN"; } return new Path(STREAM_PROTOCOL + attachmentName); }
private InputStream getAttachmentInputStream(Record record) { Object body = record.getFirstValue(Fields.ATTACHMENT_BODY); Preconditions.checkNotNull(body); if (body instanceof byte[]) { return new ByteArrayInputStream((byte[]) body); } else { return (InputStream) body; } }
private InputStream getAttachmentInputStream(Record record) { Object body = record.getFirstValue(Fields.ATTACHMENT_BODY); Preconditions.checkNotNull(body); if (body instanceof byte[]) { return new ByteArrayInputStream((byte[]) body); } else { return (InputStream) body; } }
@Override protected boolean doProcess(Record inputRecord) { JsonNode datum = (JsonNode) inputRecord.getFirstValue(Fields.ATTACHMENT_BODY); Preconditions.checkNotNull(datum); Record outputRecord = inputRecord.copy(); for (Map.Entry<String, Collection<String>> entry : stepMap.entrySet()) { String fieldName = entry.getKey(); List<String> steps = (List<String>) entry.getValue(); extractPath(datum, fieldName, steps, outputRecord, 0); } // pass record to next command in chain: return getChild().process(outputRecord); }
@Override protected boolean doProcess(Record record) { Result result = (Result)record.getFirstValue(Fields.ATTACHMENT_BODY); Preconditions.checkNotNull(result); removeAttachments(record); for (Mapping mapping : mappings) { mapping.apply(result, record); } // pass record to next command in chain: return super.doProcess(record); }
@Override protected boolean doProcess(Record record) { Result result = (Result)record.getFirstValue(Fields.ATTACHMENT_BODY); Preconditions.checkNotNull(result); removeAttachments(record); for (Mapping mapping : mappings) { mapping.apply(result, record); } // pass record to next command in chain: return super.doProcess(record); }
@Override protected boolean doProcess(Record inputRecord) { // Preconditions.checkState(ReadAvroBuilder.AVRO_MEMORY_MIME_TYPE.equals(inputRecord.getFirstValue(Fields.ATTACHMENT_MIME_TYPE))); GenericContainer datum = (GenericContainer) inputRecord.getFirstValue(Fields.ATTACHMENT_BODY); Preconditions.checkNotNull(datum); Preconditions.checkNotNull(datum.getSchema()); Record outputRecord = inputRecord.copy(); extractTree(datum, datum.getSchema(), outputRecord, outputFieldPrefix); // pass record to next command in chain: return getChild().process(outputRecord); }
@Override protected boolean doProcess(Record inputRecord) { // Preconditions.checkState(ReadAvroBuilder.AVRO_MEMORY_MIME_TYPE.equals(inputRecord.getFirstValue(Fields.ATTACHMENT_MIME_TYPE))); GenericContainer datum = (GenericContainer) inputRecord.getFirstValue(Fields.ATTACHMENT_BODY); Preconditions.checkNotNull(datum); Preconditions.checkNotNull(datum.getSchema()); Record outputRecord = inputRecord.copy(); extractTree(datum, datum.getSchema(), outputRecord, outputFieldPrefix); // pass record to next command in chain: return getChild().process(outputRecord); }
@Override protected boolean doProcess(Record inputRecord) { Record outputRecord = inputRecord.copy(); // change key Text myTextKey = (Text)inputRecord.getFirstValue(this.keyField); outputRecord.replaceValues(this.keyField, MyWritable.keyStr(myTextKey)); // change value MyWritable myWritableValue = (MyWritable)inputRecord.getFirstValue(this.valueField); outputRecord.replaceValues(this.valueField, MyWritable.valueStr(myWritableValue)); return super.doProcess(outputRecord); } }
@Test public void testIfThenElseWithElseEmpty() throws Exception { morphline = createMorphline("test-morphlines/ifThenElseWithElseEmpty"); Record record = createBasicRecord(); processAndVerifySuccess(record, record); assertEquals("init1", collector.getFirstRecord().getFirstValue("state")); }
@Test public void testIfThenElseWithThenEmpty() throws Exception { morphline = createMorphline("test-morphlines/ifThenElseWithThenEmpty"); Record record = createBasicRecord(); processAndVerifySuccess(record, record); assertEquals("init1", collector.getFirstRecord().getFirstValue("state")); }
@Test public void testIfThenElseWithElse() throws Exception { morphline = createMorphline("test-morphlines/ifThenElseWithElse"); Record record = createBasicRecord(); processAndVerifySuccess(record, record); assertEquals("else1", collector.getFirstRecord().getFirstValue("state")); }
@Test public void testIfThenElseWithThenAndNot() throws Exception { morphline = createMorphline("test-morphlines/ifThenElseWithThenAndNot"); Record record = createBasicRecord(); processAndVerifySuccess(record, record); assertEquals("then1", collector.getFirstRecord().getFirstValue("state")); }
@Test public void testIfThenElseWithThen() throws Exception { morphline = createMorphline("test-morphlines/ifThenElseWithThen"); Record record = createBasicRecord(); processAndVerifySuccess(record, record); assertEquals("then1", collector.getFirstRecord().getFirstValue("state")); }
private void testGenerateUUID(String suffix) throws Exception { morphline = createMorphline("test-morphlines/generateUUID" + suffix); Record record = new Record(); collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); Record actual = collector.getFirstRecord(); assertEquals(1, actual.get("id").size()); String uuid = (String) actual.getFirstValue("id"); assertEquals(36, uuid.length()); }
@Test public void testDecodeBase64() throws Exception { morphline = createMorphline("test-morphlines/decodeBase64"); Record record = new Record(); record.put("data", "SGVsbG8gV29ybGQ="); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); byte[] actual = (byte[]) collector.getFirstRecord().getFirstValue("data"); assertArrayEquals("Hello World".getBytes(Charsets.UTF_8), actual); assertSame(record, collector.getFirstRecord()); }
@Test public void testDetectMimeTypesWithDefaultMimeTypes() throws Exception { morphline = createMorphline("test-morphlines/detectMimeTypesWithDefaultMimeTypes"); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, Files.toByteArray(JPG_FILE)); startSession(); morphline.process(record); assertEquals("image/jpeg", collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_MIME_TYPE)); }
@Test public void testMimeTypeAlreadySpecifiedOnInputRemainsUnchanged() throws Exception { morphline = createMorphline("test-morphlines/detectMimeTypesWithDefaultMimeTypes"); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, Files.toByteArray(JPG_FILE)); record.put(Fields.ATTACHMENT_MIME_TYPE, "foo/bar"); startSession(); morphline.process(record); assertEquals("foo/bar", collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_MIME_TYPE)); }
@Test public void testNotWithFalseWithContinuation() throws Exception { morphline = createMorphline("test-morphlines/notWithFalseAndContinuation"); Record record = createBasicRecord(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertEquals(record, collector.getFirstRecord()); assertSame(record, collector.getFirstRecord()); assertEquals("touched", collector.getFirstRecord().getFirstValue("state")); }