@Override public void handle(byte[] row, ResultCell[] resultCells) { flowFile = session.write(flowFile, (out) -> { serializer.serialize(row, resultCells, out); }); handledRow = true; }
private FlowFile writeHitFlowFile(String json, ProcessSession session, FlowFile hitFlowFile, Map<String, String> attributes) { hitFlowFile = session.write(hitFlowFile, out -> out.write(json.getBytes())); return session.putAllAttributes(hitFlowFile, attributes); }
private FlowFile writeAggregationFlowFileContents(String name, String json, ProcessSession session, FlowFile aggFlowFile, Map<String, String> attributes) { aggFlowFile = session.write(aggFlowFile, out -> out.write(json.getBytes())); if (name != null) { aggFlowFile = session.putAttribute(aggFlowFile, "aggregation.name", name); } return session.putAllAttributes(aggFlowFile, attributes); }
private FlowFile initNewFlowFile(final ProcessSession session, final FlowFile origFF, final String tableName) throws IOException{ FlowFile flowFile = session.create(origFF); flowFile = session.putAttribute(flowFile, HBASE_TABLE_ATTR, tableName); flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), "application/json"); final AtomicReference<IOException> ioe = new AtomicReference<>(null); flowFile = session.write(flowFile, (out) -> { try{ out.write("[".getBytes()); }catch(IOException e){ ioe.set(e); } }); if (ioe.get() != null){ throw ioe.get(); } return flowFile; }
private Set<FlowFile> handleMultipartRequest(HttpServletRequest request, ProcessSession session, String foundSubject) throws IOException, IllegalStateException, ServletException { Set<FlowFile> flowFileSet = new HashSet<>(); String tempDir = System.getProperty("java.io.tmpdir"); request.setAttribute(Request.__MULTIPART_CONFIG_ELEMENT, new MultipartConfigElement(tempDir, multipartRequestMaxSize, multipartRequestMaxSize, multipartReadBufferSize)); List<Part> requestParts = ImmutableList.copyOf(request.getParts()); for (int i = 0; i < requestParts.size(); i++) { Part part = requestParts.get(i); FlowFile flowFile = session.create(); try (OutputStream flowFileOutputStream = session.write(flowFile)) { StreamUtils.copy(part.getInputStream(), flowFileOutputStream); } flowFile = saveRequestDetailsAsAttributes(request, session, foundSubject, flowFile); flowFile = savePartDetailsAsAttributes(session, part, flowFile, i, requestParts.size()); flowFileSet.add(flowFile); } return flowFileSet; }
private void writeData(final ProcessSession session, ConsumerRecord<byte[], byte[]> record, final TopicPartition topicPartition) { FlowFile flowFile = session.create(); final BundleTracker tracker = new BundleTracker(record, topicPartition, keyEncoding); tracker.incrementRecordCount(1); final byte[] value = record.value(); if (value != null) { flowFile = session.write(flowFile, out -> { out.write(value); }); } tracker.updateFlowFile(flowFile); populateAttributes(tracker); session.transfer(tracker.flowFile, REL_SUCCESS); }
private void writeData(final ProcessSession session, ConsumerRecord<byte[], byte[]> record, final TopicPartition topicPartition) { FlowFile flowFile = session.create(); final BundleTracker tracker = new BundleTracker(record, topicPartition, keyEncoding); tracker.incrementRecordCount(1); final byte[] value = record.value(); if (value != null) { flowFile = session.write(flowFile, out -> { out.write(value); }); } tracker.updateFlowFile(flowFile); populateAttributes(tracker); session.transfer(tracker.flowFile, REL_SUCCESS); }
private RecordSetWriter createIfNecessary(final RecordSetWriter writer, final RecordSetWriterFactory factory, final ProcessSession session, final FlowFile flowFile, final RecordSchema outputSchema) throws SchemaNotFoundException, IOException { if (writer != null) { return writer; } final OutputStream out = session.write(flowFile); final RecordSetWriter created = factory.createWriter(getLogger(), outputSchema, out); created.beginRecordSet(); return created; }
private void writeData(final ProcessSession session, ConsumerRecord<byte[], byte[]> record, final TopicPartition topicPartition) { FlowFile flowFile = session.create(); final BundleTracker tracker = new BundleTracker(record, topicPartition, keyEncoding); tracker.incrementRecordCount(1); final byte[] value = record.value(); if (value != null) { flowFile = session.write(flowFile, out -> { out.write(value); }); } flowFile = session.putAllAttributes(flowFile, getAttributes(record)); tracker.updateFlowFile(flowFile); populateAttributes(tracker); session.transfer(tracker.flowFile, REL_SUCCESS); }
private void writeData(final ProcessSession session, ConsumerRecord<byte[], byte[]> record, final TopicPartition topicPartition) { FlowFile flowFile = session.create(); final BundleTracker tracker = new BundleTracker(record, topicPartition, keyEncoding); tracker.incrementRecordCount(1); final byte[] value = record.value(); if (value != null) { flowFile = session.write(flowFile, out -> { out.write(value); }); } flowFile = session.putAllAttributes(flowFile, getAttributes(record)); tracker.updateFlowFile(flowFile); populateAttributes(tracker); session.transfer(tracker.flowFile, REL_SUCCESS); }
private void writeData(final ProcessSession session, ConsumerRecord<byte[], byte[]> record, final TopicPartition topicPartition) { FlowFile flowFile = session.create(); final BundleTracker tracker = new BundleTracker(record, topicPartition, keyEncoding); tracker.incrementRecordCount(1); final byte[] value = record.value(); if (value != null) { flowFile = session.write(flowFile, out -> { out.write(value); }); } flowFile = session.putAllAttributes(flowFile, getAttributes(record)); tracker.updateFlowFile(flowFile); populateAttributes(tracker); session.transfer(tracker.flowFile, REL_SUCCESS); }
private void writeFlowFiles(PartitionContext context, Iterable<EventData> messages, ProcessSession session, StopWatch stopWatch) { final String eventHubName = context.getEventHubPath(); final String partitionId = context.getPartitionId(); final String consumerGroup = context.getConsumerGroupName(); messages.forEach(eventData -> { FlowFile flowFile = session.create(); final Map<String, String> attributes = new HashMap<>(); putEventHubAttributes(attributes, eventHubName, partitionId, eventData); flowFile = session.putAllAttributes(flowFile, attributes); flowFile = session.write(flowFile, out -> { out.write(eventData.getBytes()); }); transferTo(REL_SUCCESS, session, stopWatch, eventHubName, partitionId, consumerGroup, flowFile); }); }
public void handle(FlowFile original, ProcessSession processSession, String chunkName, byte[] badChunk) { FlowFile flowFile = processSession.create(original); flowFile = processSession.putAttribute(flowFile, CoreAttributes.FILENAME.key(), chunkName); flowFile = processSession.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), MediaType.APPLICATION_BINARY.toString()); flowFile = processSession.write(flowFile, out -> out.write(badChunk)); processSession.transfer(flowFile, badChunkRelationship); } }
protected static void transferEvent(final Event event, ProcessSession session, Relationship relationship) { FlowFile flowFile = session.create(); flowFile = session.putAllAttributes(flowFile, event.getHeaders()); flowFile = session.write(flowFile, new OutputStreamCallback() { @Override public void process(final OutputStream out) throws IOException { out.write(event.getBody()); } }); session.getProvenanceReporter() .create(flowFile); session.transfer(flowFile, relationship); }
@Override public void accept(final JMSResponse response) { if (response == null) { return; } FlowFile flowFile = processSession.create(); flowFile = processSession.write(flowFile, out -> out.write(response.getMessageBody())); final Map<String, String> jmsHeaders = response.getMessageHeaders(); final Map<String, String> jmsProperties = response.getMessageProperties(); flowFile = ConsumeJMS.this.updateFlowFileAttributesWithJMSAttributes(jmsHeaders, flowFile, processSession); flowFile = ConsumeJMS.this.updateFlowFileAttributesWithJMSAttributes(jmsProperties, flowFile, processSession); flowFile = processSession.putAttribute(flowFile, JMS_SOURCE_DESTINATION_NAME, destinationName); processSession.getProvenanceReporter().receive(flowFile, destinationName); processSession.transfer(flowFile, REL_SUCCESS); processSession.commit(); } });
@Override public long writeEvent(ProcessSession session, String transitUri, DDLEventInfo eventInfo, long currentSequenceId, Relationship relationship) { FlowFile flowFile = session.create(); flowFile = session.write(flowFile, (outputStream) -> { super.startJson(outputStream, eventInfo); super.writeJson(eventInfo); jsonGenerator.writeStringField("query", eventInfo.getQuery()); super.endJson(); }); flowFile = session.putAllAttributes(flowFile, getCommonAttributes(currentSequenceId, eventInfo)); session.transfer(flowFile, relationship); session.getProvenanceReporter().receive(flowFile, transitUri); return currentSequenceId + 1; } }
@Override public long writeEvent(ProcessSession session, String transitUri, T eventInfo, long currentSequenceId, Relationship relationship) { FlowFile flowFile = session.create(); flowFile = session.write(flowFile, (outputStream) -> { super.startJson(outputStream, eventInfo); writeJson(eventInfo); // Nothing in the body super.endJson(); }); flowFile = session.putAllAttributes(flowFile, getCommonAttributes(currentSequenceId, eventInfo)); session.transfer(flowFile, relationship); session.getProvenanceReporter().receive(flowFile, transitUri); return currentSequenceId + 1; } }
@Override public long writeEvent(ProcessSession session, String transitUri, T eventInfo, long currentSequenceId, Relationship relationship) { FlowFile flowFile = session.create(); flowFile = session.write(flowFile, (outputStream) -> { super.startJson(outputStream, eventInfo); writeJson(eventInfo); // Nothing in the body super.endJson(); }); flowFile = session.putAllAttributes(flowFile, getCommonAttributes(currentSequenceId, eventInfo)); session.transfer(flowFile, relationship); session.getProvenanceReporter().receive(flowFile, transitUri); return currentSequenceId + 1; } }
private void handleParseFailure(final StandardEvent event, final ProcessSession session, final Exception cause, final String message) { // If we are unable to parse the data, we need to transfer it to 'parse failure' relationship final Map<String, String> attributes = getAttributes(event.getSender()); FlowFile failureFlowFile = session.create(); failureFlowFile = session.write(failureFlowFile, out -> out.write(event.getData())); failureFlowFile = session.putAllAttributes(failureFlowFile, attributes); final String transitUri = getTransitUri(event.getSender()); session.getProvenanceReporter().receive(failureFlowFile, transitUri); session.transfer(failureFlowFile, REL_PARSE_FAILURE); if (cause == null) { getLogger().error(message); } else { getLogger().error(message, cause); } session.adjustCounter("Parse Failures", 1, false); }
@Test public void testSplit_pathToNullValue() throws Exception { final TestRunner testRunner = TestRunners.newTestRunner(new SplitJson()); testRunner.setProperty(SplitJson.ARRAY_JSON_PATH_EXPRESSION, "$.nullField"); ProcessSession session = testRunner.getProcessSessionFactory().createSession(); FlowFile ff = session.create(); ff = session.write(ff, new OutputStreamCallback() { @Override public void process(OutputStream out) throws IOException { try (OutputStream outputStream = new BufferedOutputStream(out)) { outputStream.write("{\"stringField\": \"String Value\", \"nullField\": null}".getBytes(StandardCharsets.UTF_8)); } } }); testRunner.enqueue(ff); testRunner.run(); testRunner.assertTransferCount(SplitJson.REL_FAILURE, 1); }