public StandardDataPacket(final Map<String, String> attributes, final InputStream stream, final long size) { this.attributes = attributes; this.stream = new MinimumLengthInputStream(new LimitingInputStream(stream, size), size); this.size = size; }
private static void copy(final File input, final File destination, final long length) throws IOException { try (final InputStream fis = new FileInputStream(input); final LimitingInputStream in = new LimitingInputStream(fis, length); final OutputStream fos = new FileOutputStream(destination)) { StreamUtils.copy(in, fos); } }
final InputStream serdeHeaderIn = new LimitingInputStream(in, serdeHeaderLength); final DataInputStream dis = new DataInputStream(serdeHeaderIn); serde.readHeader(dis);
limitedStream = rawInputStream; } else { limitedStream = new LimitingInputStream(rawInputStream, offset1 - rawInputStream.getBytesConsumed());
private void resetStreamForNextBlock() throws IOException { final InputStream limitedStream; if (tocReader == null) { limitedStream = rawInputStream; } else { final long offset = tocReader.getBlockOffset(1 + getBlockIndex()); if (offset < 0) { limitedStream = rawInputStream; } else { limitedStream = new LimitingInputStream(rawInputStream, offset - rawInputStream.getBytesConsumed()); } } final InputStream readableStream; if (compressed) { readableStream = new BufferedInputStream(new GZIPInputStream(limitedStream)); } else { readableStream = new BufferedInputStream(limitedStream); } byteCountingIn = new ByteCountingInputStream(readableStream, rawInputStream.getBytesConsumed()); dis = new DataInputStream(byteCountingIn); }
@Override protected StandardProvenanceEventRecord nextRecord(final DataInputStream in, final int serializationVersion) throws IOException { verifySerializationVersion(serializationVersion); final long byteOffset = getBytesConsumed(); final int recordLength = in.readInt(); final InputStream limitedIn = new LimitingInputStream(in, recordLength); final Record eventRecord = recordReader.readRecord(limitedIn); if (eventRecord == null) { return null; } return EventRecord.getEvent(eventRecord, getFilename(), byteOffset, getMaxAttributeLength()); }
private RemoteFlowFileRecord receiveFlowFile(final DataInputStream dis, final OutputStream out, final ContentClaim contentClaim, final long claimOffset, final int protocolVersion, final String peerDescription, final LoadBalanceCompression compression) throws IOException { final int metadataLength = dis.readInt(); DataInputStream metadataIn = new DataInputStream(new LimitingInputStream(dis, metadataLength)); if (compression != LoadBalanceCompression.DO_NOT_COMPRESS) { metadataIn = new DataInputStream(new GZIPInputStream(metadataIn)); } final Map<String, String> attributes = readAttributes(metadataIn); final String sourceSystemUuid = attributes.get(CoreAttributes.UUID.key()); logger.debug("Received Attributes {} from Peer {}", attributes, peerDescription); final long lineageStartDate = metadataIn.readLong(); final long entryDate = metadataIn.readLong(); final ContentClaimTriple contentClaimTriple = consumeContent(dis, out, contentClaim, claimOffset, peerDescription, compression == LoadBalanceCompression.COMPRESS_ATTRIBUTES_AND_CONTENT); final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder() .id(flowFileRepository.getNextFlowFileSequence()) .addAttributes(attributes) .addAttribute(CoreAttributes.UUID.key(), UUID.randomUUID().toString()) .contentClaim(contentClaimTriple.getContentClaim()) .contentClaimOffset(contentClaimTriple.getClaimOffset()) .size(contentClaimTriple.getContentLength()) .entryDate(entryDate) .lineageStart(lineageStartDate, lineageStartIndex.getAndIncrement()) .build(); logger.debug("Received FlowFile {} with {} attributes and {} bytes of content", flowFileRecord, attributes.size(), contentClaimTriple.getContentLength()); return new RemoteFlowFileRecord(sourceSystemUuid, flowFileRecord); }
private StandardProvenanceEventRecord readRecord(final DataInputStream in, final long eventId, final long startOffset, final int recordLength) throws IOException { final InputStream limitedIn = new LimitingInputStream(in, recordLength); final Record eventRecord = recordReader.readRecord(limitedIn); if (eventRecord == null) { return null; } final StandardProvenanceEventRecord deserializedEvent = LookupTableEventRecord.getEvent(eventRecord, getFilename(), startOffset, getMaxAttributeLength(), firstEventId, systemTimeOffset, componentIds, componentTypes, queueIds, eventTypes); deserializedEvent.setEventId(eventId); return deserializedEvent; }
private StandardProvenanceEventRecord readRecord(final DataInputStream inputStream, final long eventId, final long startOffset, final int recordLength) throws IOException { try { final InputStream limitedIn = new LimitingInputStream(inputStream, recordLength); byte[] encryptedSerializedBytes = new byte[recordLength]; DataInputStream encryptedInputStream = new DataInputStream(limitedIn); encryptedInputStream.readFully(encryptedSerializedBytes); byte[] plainSerializedBytes = decrypt(encryptedSerializedBytes, Long.toString(eventId)); InputStream plainStream = new ByteArrayInputStream(plainSerializedBytes); final Record eventRecord = getRecordReader().readRecord(plainStream); if (eventRecord == null) { return null; } final StandardProvenanceEventRecord deserializedEvent = LookupTableEventRecord.getEvent(eventRecord, getFilename(), startOffset, getMaxAttributeLength(), getFirstEventId(), getSystemTimeOffset(), getComponentIds(), getComponentTypes(), getQueueIds(), getEventTypes()); deserializedEvent.setEventId(eventId); return deserializedEvent; } catch (EncryptionException e) { logger.error("Encountered an error reading the record: ", e); throw new IOException(e); } }
@Override public void data(final InputStream data) throws RejectException, TooMuchDataException, IOException { final ProcessSession processSession = sessionFactory.createSession(); final StopWatch watch = new StopWatch(); watch.start(); try { FlowFile flowFile = processSession.create(); final AtomicBoolean limitExceeded = new AtomicBoolean(false); flowFile = processSession.write(flowFile, (OutputStream out) -> { final LimitingInputStream lis = new LimitingInputStream(data, maxMessageSize); IOUtils.copy(lis, out); if (lis.hasReachedLimit()) { limitExceeded.set(true); } }); if (limitExceeded.get()) { throw new TooMuchDataException("Maximum message size limit reached - client must send smaller messages"); } flowFile = processSession.putAllAttributes(flowFile, extractMessageAttributes()); watch.stop(); processSession.getProvenanceReporter().receive(flowFile, "smtp://" + host + ":" + port + "/", watch.getDuration(TimeUnit.MILLISECONDS)); processSession.transfer(flowFile, ListenSMTP.REL_SUCCESS); processSession.commit(); } catch (FlowFileAccessException | IllegalStateException | RejectException | IOException ex) { log.error("Unable to fully process input due to " + ex.getMessage(), ex); throw ex; } finally { processSession.rollback(); //make sure this happens no matter what - is safe } }
stream = new LimitingInputStream(stream, flowFile.getSize());
@Override public void data(final InputStream data) throws RejectException, TooMuchDataException, IOException { final ProcessSession processSession = sessionFactory.createSession(); final StopWatch watch = new StopWatch(); watch.start(); try { FlowFile flowFile = processSession.create(); final AtomicBoolean limitExceeded = new AtomicBoolean(false); flowFile = processSession.write(flowFile, (OutputStream out) -> { final LimitingInputStream lis = new LimitingInputStream(data, maxMessageSize); IOUtils.copy(lis, out); if (lis.hasReachedLimit()) { limitExceeded.set(true); } }); if (limitExceeded.get()) { throw new TooMuchDataException("Maximum message size limit reached - client must send smaller messages"); } flowFile = processSession.putAllAttributes(flowFile, extractMessageAttributes()); watch.stop(); processSession.getProvenanceReporter().receive(flowFile, "smtp://" + host + ":" + port + "/", watch.getDuration(TimeUnit.MILLISECONDS)); processSession.transfer(flowFile, ListenSMTP.REL_SUCCESS); processSession.commit(); } catch (FlowFileAccessException | IllegalStateException | RejectException | IOException ex) { log.error("Unable to fully process input due to " + ex.getMessage(), ex); throw ex; } finally { processSession.rollback(); //make sure this happens no matter what - is safe } }