@Override public void process(OutputStream out) throws IOException { try (InputStream is = session.read(flowFile)) { IOUtils.copy(is, out); } } });
static DocumentContext validateAndEstablishJsonContext(ProcessSession processSession, FlowFile flowFile) { // Parse the document once into an associated context to support multiple path evaluations if specified final AtomicReference<DocumentContext> contextHolder = new AtomicReference<>(null); processSession.read(flowFile, new InputStreamCallback() { @Override public void process(InputStream in) throws IOException { try (BufferedInputStream bufferedInputStream = new BufferedInputStream(in)) { DocumentContext ctx = JsonPath.using(STRICT_PROVIDER_CONFIGURATION).parse(bufferedInputStream); contextHolder.set(ctx); } } }); return contextHolder.get(); }
private String extractTextMessageBody(FlowFile flowFile, ProcessSession session, String charset) { final StringWriter writer = new StringWriter(); session.read(flowFile, in -> IOUtils.copy(in, writer, Charset.forName(charset))); return writer.toString(); } }
/** * Extracts contents of the {@link FlowFile} to byte array. */ private byte[] extractMessage(FlowFile flowFile, ProcessSession processSession) { final byte[] messageContent = new byte[(int) flowFile.getSize()]; processSession.read(flowFile, new InputStreamCallback() { @Override public void process(final InputStream in) throws IOException { StreamUtils.fillBuffer(in, messageContent, true); } }); return messageContent; }
/** * Extracts contents of the {@link FlowFile} as byte array. */ private byte[] extractMessage(FlowFile flowFile, ProcessSession session){ final byte[] messageContent = new byte[(int) flowFile.getSize()]; session.read(flowFile, new InputStreamCallback() { @Override public void process(final InputStream in) throws IOException { StreamUtils.fillBuffer(in, messageContent, true); } }); return messageContent; }
/** * Executes the given callback against the contents corresponding to the * given FlowFile. * * @param flowFile flow file to retrieve content of * @param reader callback that will be called to read the flow file content * @throws FlowFileAccessException if some IO problem occurs accessing FlowFile content; if an attempt is made to access the InputStream provided to the given InputStreamCallback after * this method completed its execution */ @Override public void read(FlowFile flowFile, InputStreamCallback reader) throws FlowFileAccessException { flowFile = unwrap(flowFile); s.read(flowFile, reader); }
/** * Determines the CQL statement that should be executed for the given FlowFile * * @param session the session that can be used to access the given FlowFile * @param flowFile the FlowFile whose CQL statement should be executed * @return the CQL that is associated with the given FlowFile */ private String getCQL(final ProcessSession session, final FlowFile flowFile, final Charset charset) { // Read the CQL from the FlowFile's content final byte[] buffer = new byte[(int) flowFile.getSize()]; session.read(flowFile, new InputStreamCallback() { @Override public void process(final InputStream in) throws IOException { StreamUtils.fillBuffer(in, buffer); } }); // Create the PreparedStatement string to use for this FlowFile. return new String(buffer, charset); }
/** * Determines the HiveQL statement that should be executed for the given FlowFile * * @param session the session that can be used to access the given FlowFile * @param flowFile the FlowFile whose HiveQL statement should be executed * @return the HiveQL that is associated with the given FlowFile */ protected String getHiveQL(final ProcessSession session, final FlowFile flowFile, final Charset charset) { // Read the HiveQL from the FlowFile's content final byte[] buffer = new byte[(int) flowFile.getSize()]; session.read(flowFile, new InputStreamCallback() { @Override public void process(final InputStream in) throws IOException { StreamUtils.fillBuffer(in, buffer); } }); // Create the PreparedStatement to use for this FlowFile. return new String(buffer, charset); }
/** * Helper method to read the FlowFile content stream into a byte array. * * @param session * - the current process session. * @param flowFile * - the FlowFile to read the content from. * * @return byte array representation of the FlowFile content. */ protected byte[] readContent(final ProcessSession session, final FlowFile flowFile) { final ByteArrayOutputStream baos = new ByteArrayOutputStream((int) flowFile.getSize() + 1); session.read(flowFile, new InputStreamCallback() { @Override public void process(final InputStream in) throws IOException { StreamUtils.copy(in, baos); } }); return baos.toByteArray(); } }
/** * Provides an InputStream that can be used to read the contents of the given FlowFile. * This method differs from those that make use of callbacks in that this method returns * an InputStream and expects the caller to properly handle the lifecycle of the InputStream * (i.e., the caller is responsible for ensuring that the InputStream is closed appropriately). * The Process Session may or may not handle closing the stream when {@link #commit()} or {@link #rollback()} * is called, but the responsibility of doing so belongs to the caller. The InputStream will throw * an IOException if an attempt is made to read from the stream after the session is committed or * rolled back. * * @param flowFile the FlowFile to read * @return an InputStream that can be used to read the contents of the FlowFile * @throws IllegalStateException if detected that this method is being called from within a callback of another method in this session and for the given FlowFile(s) * @throws FlowFileHandlingException if the given FlowFile is already transferred or removed or doesn't belong to this session. Automatic rollback will occur. * @throws MissingFlowFileException if the given FlowFile content cannot be found. The FlowFile should no longer be referenced, will be internally destroyed, and the session is automatically * rolled back and what is left of the FlowFile is destroyed. */ @Override public InputStream read(FlowFile flowFile) { flowFile = unwrap(flowFile); return s.read(flowFile); }
/** * Determines the SQL statement that should be executed for the given FlowFile * * @param session the session that can be used to access the given FlowFile * @param flowFile the FlowFile whose SQL statement should be executed * * @return the SQL that is associated with the given FlowFile */ private String getSQL(final ProcessSession session, final FlowFile flowFile) { // Read the SQL from the FlowFile's content final byte[] buffer = new byte[(int) flowFile.getSize()]; session.read(flowFile, new InputStreamCallback() { @Override public void process(final InputStream in) throws IOException { StreamUtils.fillBuffer(in, buffer); } }); // Create the PreparedStatement to use for this FlowFile. final String sql = new String(buffer, StandardCharsets.UTF_8); return sql; }
/** * Executes the given callback against the contents corresponding to the * given FlowFile. * <p> * <i>Note</i>: The OutputStream provided to the given OutputStreamCallback * will not be accessible once this method has completed its execution. * * @param flowFile flow file to retrieve content of * @param allowSessionStreamManagement allow session to hold the stream open for performance reasons * @param reader that will be called to read the flow file content * @throws IllegalStateException if detected that this method is being called from within a callback of another method in this session and for the given FlowFile(s) * @throws FlowFileHandlingException if the given FlowFile is already transferred or removed or doesn't belong to this session. Automatic rollback will occur. * @throws MissingFlowFileException if the given FlowFile content cannot be found. The FlowFile should no longer be reference, will be internally destroyed, and the session is automatically * rolled back and what is left of the FlowFile is destroyed. * @throws FlowFileAccessException if some IO problem occurs accessing FlowFile content; if an attempt is made to access the InputStream provided to the given InputStreamCallback after this * method completed its execution */ @Override public void read(FlowFile flowFile, boolean allowSessionStreamManagement, InputStreamCallback reader) throws FlowFileAccessException { flowFile = unwrap(flowFile); s.read(flowFile, allowSessionStreamManagement, reader); }
/** * Determines the HiveQL statement that should be executed for the given FlowFile * * @param session the session that can be used to access the given FlowFile * @param flowFile the FlowFile whose HiveQL statement should be executed * @return the HiveQL that is associated with the given FlowFile */ protected String getHiveQL(final ProcessSession session, final FlowFile flowFile, final Charset charset) { // Read the HiveQL from the FlowFile's content final byte[] buffer = new byte[(int) flowFile.getSize()]; session.read(flowFile, in -> StreamUtils.fillBuffer(in, buffer)); // Create the PreparedStatement to use for this FlowFile. return new String(buffer, charset); }
/** * Extracts contents of the {@link FlowFile} as byte array. */ private byte[] extractMessageBody(FlowFile flowFile, ProcessSession session) { final byte[] messageContent = new byte[(int) flowFile.getSize()]; session.read(flowFile, in -> StreamUtils.fillBuffer(in, messageContent, true)); return messageContent; }
@Override public byte[] getBody() { synchronized (bodyLock) { if (!bodyLoaded) { if (flowFile.getSize() > Integer.MAX_VALUE) { throw new RuntimeException("Can't get body of Event because the backing FlowFile is too large (" + flowFile.getSize() + " bytes)"); } final ByteArrayOutputStream baos = new ByteArrayOutputStream((int) flowFile.getSize()); session.read(flowFile, new InputStreamCallback() { @Override public void process(InputStream in) throws IOException { try (BufferedInputStream input = new BufferedInputStream(in)) { StreamUtils.copy(input, baos); } baos.close(); } }); body = baos.toByteArray(); bodyLoaded = true; } } return body; }
@Override public void reset() { if (rawIn != null) { try { rawIn.close(); } catch (final Exception e) { logger.warn("Could not close FlowFile's input due to " + e, e); } } rawIn = session.read(flowFile); try { recordParser = recordParserFactory.createRecordReader(flowFile, rawIn, logger); } catch (final Exception e) { throw new ProcessException("Failed to reset stream", e); } }
private String getMessage(final FlowFile flowFile, final ProcessContext context, final ProcessSession session) { String messageText = ""; if(context.getProperty(CONTENT_AS_MESSAGE).evaluateAttributeExpressions(flowFile).asBoolean()) { // reading all the content of the input flow file final byte[] byteBuffer = new byte[(int) flowFile.getSize()]; session.read(flowFile, new InputStreamCallback() { @Override public void process(InputStream in) throws IOException { StreamUtils.fillBuffer(in, byteBuffer, false); } }); messageText = new String(byteBuffer, 0, byteBuffer.length, Charset.forName("UTF-8")); } else if (context.getProperty(MESSAGE).isSet()) { messageText = context.getProperty(MESSAGE).evaluateAttributeExpressions(flowFile).getValue(); } if (context.getProperty(INCLUDE_ALL_ATTRIBUTES).asBoolean()) { return formatAttributes(flowFile, messageText); } return messageText; }
private void binFlowFile(final ProcessContext context, final FlowFile flowFile, final ProcessSession session, final RecordBinManager binManager, final boolean block) { final RecordReaderFactory readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class); try (final InputStream in = session.read(flowFile); final RecordReader reader = readerFactory.createRecordReader(flowFile, in, getLogger())) { final RecordSchema schema = reader.getSchema(); final String groupId = getGroupId(context, flowFile, schema, session); getLogger().debug("Got Group ID {} for {}", new Object[] {groupId, flowFile}); binManager.add(groupId, flowFile, reader, session, block); } catch (MalformedRecordException | IOException | SchemaNotFoundException e) { throw new ProcessException(e); } }
@Test public void testMap2FlowFileTextMessage() throws Exception { TestRunner runner = TestRunners.newTestRunner(GetJMSQueue.class); TextMessage textMessage = new ActiveMQTextMessage(); String payload = "Hello world!"; textMessage.setText(payload); ProcessContext context = runner.getProcessContext(); ProcessSession session = runner.getProcessSessionFactory().createSession(); ProcessorInitializationContext pic = new MockProcessorInitializationContext(runner.getProcessor(), (MockProcessContext) runner.getProcessContext()); JmsProcessingSummary summary = JmsConsumer.map2FlowFile(context, session, textMessage, true, pic.getLogger()); assertEquals("TextMessage content length should equal to FlowFile content size", payload.length(), summary.getLastFlowFile().getSize()); final byte[] buffer = new byte[payload.length()]; runner.clearTransferState(); session.read(summary.getLastFlowFile(), new InputStreamCallback() { @Override public void process(InputStream in) throws IOException { StreamUtils.fillBuffer(in, buffer, false); } }); String contentString = new String(buffer, "UTF-8"); assertEquals("", payload, contentString); }
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { FlowFile flowFile = session.get(); if (flowFile == null) { return; } final StopWatch stopWatch = new StopWatch(true); final byte[] buffer = new byte[(int) flowFile.getSize()]; session.read(flowFile, in -> StreamUtils.fillBuffer(in, buffer)); try { sendMessage(buffer); } catch (final ProcessException processException) { getLogger().error("Failed to send {} to EventHub due to {}; routing to failure", new Object[]{flowFile, processException}, processException); session.transfer(session.penalize(flowFile), REL_FAILURE); return; } final String namespace = context.getProperty(NAMESPACE).getValue(); final String eventHubName = context.getProperty(EVENT_HUB_NAME).getValue(); session.getProvenanceReporter().send(flowFile, "amqps://" + namespace + ".servicebus.windows.net" + "/" + eventHubName, stopWatch.getElapsed(TimeUnit.MILLISECONDS)); session.transfer(flowFile, REL_SUCCESS); }