@Override public Map<String, String> getAttributesToAdd() { Map<String, String> attributesToAdd = new HashMap<>(); attributesToAdd.put(CoreAttributes.MIME_TYPE.key(), mimeType); // Add any attributes from the record writer (if present) final WriteResult result = writeResultRef.get(); if (result != null) { if (result.getAttributes() != null) { attributesToAdd.putAll(result.getAttributes()); } attributesToAdd.put("record.count", String.valueOf(result.getRecordCount())); } return attributesToAdd; }
private void completeFlowFile(final ProcessSession session, final FlowFile flowFile, final RecordSetWriter writer, final Relationship relationship, final String details) throws IOException { final WriteResult writeResult = writer.finishRecordSet(); writer.close(); final Map<String, String> attributes = new HashMap<>(); attributes.putAll(writeResult.getAttributes()); attributes.put("record.count", String.valueOf(writeResult.getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); session.putAllAttributes(flowFile, attributes); session.transfer(flowFile, relationship); session.getProvenanceReporter().route(flowFile, relationship, details); }
protected byte[] getData(final ReportingContext context, InputStream in, Map<String, String> attributes) { try (final JsonRecordReader reader = new JsonRecordReader(in, recordSchema)) { final RecordSetWriterFactory writerFactory = context.getProperty(RECORD_WRITER).asControllerService(RecordSetWriterFactory.class); final RecordSchema writeSchema = writerFactory.getSchema(null, recordSchema); final ByteArrayOutputStream out = new ByteArrayOutputStream(); try (final RecordSetWriter writer = writerFactory.createWriter(getLogger(), writeSchema, out)) { writer.beginRecordSet(); Record record; while ((record = reader.nextRecord()) != null) { writer.write(record); } final WriteResult writeResult = writer.finishRecordSet(); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); attributes.putAll(writeResult.getAttributes()); } return out.toByteArray(); } catch (IOException | SchemaNotFoundException | MalformedRecordException e) { throw new ProcessException("Failed to write metrics using record writer: " + e.getMessage(), e); } }
private boolean processBundle(final BundleTracker bundle) throws IOException { final RecordSetWriter writer = bundle.recordWriter; if (writer != null) { final WriteResult writeResult; try { writeResult = writer.finishRecordSet(); } finally { writer.close(); } if (writeResult.getRecordCount() == 0) { getProcessSession().remove(bundle.flowFile); return false; } final Map<String, String> attributes = new HashMap<>(); attributes.putAll(writeResult.getAttributes()); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); bundle.flowFile = getProcessSession().putAllAttributes(bundle.flowFile, attributes); } populateAttributes(bundle); return true; }
try (final RecordSetWriter writer = writerFactory.createWriter(logger, schema, baos)) { final WriteResult writeResult = writer.write(record); additionalAttributes = writeResult.getAttributes(); writer.flush();
private boolean processBundle(final BundleTracker bundle) throws IOException { final RecordSetWriter writer = bundle.recordWriter; if (writer != null) { final WriteResult writeResult; try { writeResult = writer.finishRecordSet(); } finally { writer.close(); } if (writeResult.getRecordCount() == 0) { getProcessSession().remove(bundle.flowFile); return false; } final Map<String, String> attributes = new HashMap<>(); attributes.putAll(writeResult.getAttributes()); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); bundle.flowFile = getProcessSession().putAllAttributes(bundle.flowFile, attributes); } populateAttributes(bundle); return true; }
private boolean processBundle(final BundleTracker bundle) throws IOException { final RecordSetWriter writer = bundle.recordWriter; if (writer != null) { final WriteResult writeResult; try { writeResult = writer.finishRecordSet(); } finally { writer.close(); } if (writeResult.getRecordCount() == 0) { getProcessSession().remove(bundle.flowFile); return false; } final Map<String, String> attributes = new HashMap<>(); attributes.putAll(writeResult.getAttributes()); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); bundle.flowFile = getProcessSession().putAllAttributes(bundle.flowFile, attributes); } populateAttributes(bundle); return true; }
private boolean processBundle(final BundleTracker bundle) throws IOException { final RecordSetWriter writer = bundle.recordWriter; if (writer != null) { final WriteResult writeResult; try { writeResult = writer.finishRecordSet(); } finally { writer.close(); } if (writeResult.getRecordCount() == 0) { getProcessSession().remove(bundle.flowFile); return false; } final Map<String, String> attributes = new HashMap<>(); attributes.putAll(writeResult.getAttributes()); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); bundle.flowFile = getProcessSession().putAllAttributes(bundle.flowFile, attributes); } populateAttributes(bundle); return true; }
try (final RecordSetWriter writer = writerFactory.createWriter(logger, schema, baos)) { final WriteResult writeResult = writer.write(record); additionalAttributes = writeResult.getAttributes(); writer.flush();
try (final RecordSetWriter writer = writerFactory.createWriter(logger, schema, baos)) { final WriteResult writeResult = writer.write(record); additionalAttributes = writeResult.getAttributes(); writer.flush();
attributes.putAll(mergedAttributes); attributes.putAll(writeResult.getAttributes()); attributes.put("record.count", String.valueOf(writeResult.getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), recordWriter.getMimeType());
@Override public void process(final InputStream in, final OutputStream out) throws IOException { try (final RecordReader reader = readerFactory.createRecordReader(originalAttributes, in, getLogger())) { final RecordSchema writeSchema = writerFactory.getSchema(originalAttributes, reader.getSchema()); try (final RecordSetWriter writer = writerFactory.createWriter(getLogger(), writeSchema, out)) { writer.beginRecordSet(); Record record; while ((record = reader.nextRecord()) != null) { final Record processed = AbstractRecordProcessor.this.process(record, writeSchema, original, context); writer.write(processed); } final WriteResult writeResult = writer.finishRecordSet(); attributes.put("record.count", String.valueOf(writeResult.getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); attributes.putAll(writeResult.getAttributes()); recordCount.set(writeResult.getRecordCount()); } } catch (final SchemaNotFoundException e) { throw new ProcessException(e.getLocalizedMessage(), e); } catch (final MalformedRecordException e) { throw new ProcessException("Could not parse incoming data", e); } } });
attributes.put(FRAGMENT_ID, fragmentId); attributes.put(SEGMENT_ORIGINAL_FILENAME, original.getAttribute(CoreAttributes.FILENAME.key())); attributes.putAll(writeResult.getAttributes());
attributes.putAll(writeResult.getAttributes()); attributes.put("record.count", String.valueOf(writeResult.getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType());
final Map<String,String> attributes = new HashMap<>(writeResult.get().getAttributes()); attributes.put(CoreAttributes.FILENAME.key(), newFilename); attributes.put(ABSOLUTE_HDFS_PATH_ATTRIBUTE, hdfsPath);
final Map<String,String> attributes = new HashMap<>(writeResult.get().getAttributes()); attributes.put(RECORD_COUNT_ATTR, String.valueOf(writeResult.get().getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), mimeTypeRef.get());
attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); if (lastWriteResult != null) { attributes.putAll(lastWriteResult.getAttributes());
attributes.put("record.count", String.valueOf(writeResult.getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), recordSetWriter.getMimeType()); attributes.putAll(writeResult.getAttributes()); session.transfer(session.putAllAttributes(outFlowFile, attributes), REL_FORK);
attributes.put("record.count", String.valueOf(writeResult.getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); attributes.putAll(writeResult.getAttributes());
attributes.putAll(writeResult.getAttributes()); } catch (Exception e) { logger.error("Unable to write transformed records {} due to {}", new Object[]{original, e.toString(), e});