@Override public Object getFieldValue(final String fieldName) { switch (fieldName) { case FlowFileSchema.ATTRIBUTES: return flowFile.getAttributes(); case FlowFileSchema.CONTENT_CLAIM: return contentClaim; case FlowFileSchema.ENTRY_DATE: return flowFile.getEntryDate(); case FlowFileSchema.FLOWFILE_SIZE: return flowFile.getSize(); case FlowFileSchema.LINEAGE_START_DATE: return flowFile.getLineageStartDate(); case FlowFileSchema.LINEAGE_START_INDEX: return flowFile.getLineageStartIndex(); case FlowFileSchema.QUEUE_DATE: return flowFile.getLastQueueDate(); case FlowFileSchema.QUEUE_DATE_INDEX: return flowFile.getQueueDateIndex(); case FlowFileSchema.RECORD_ID: return flowFile.getId(); } return null; }
final InputStream rawIn; try { rawIn = contentRepository.read(flowFile.getContentClaim()); } catch (final ContentNotFoundException cnfe) { throw new ContentNotFoundException(flowFile, flowFile.getContentClaim(), cnfe.getMessage()); if (flowFile.getContentClaimOffset() > 0) { try { StreamUtils.skip(rawIn, flowFile.getContentClaimOffset()); } catch (final EOFException eof) { throw new ContentNotFoundException(flowFile, flowFile.getContentClaim(), "FlowFile has a Content Claim Offset of " + flowFile.getContentClaimOffset() + " bytes but the Content Claim does not have that many bytes"); final InputStream limitedIn = new LimitedInputStream(rawIn, flowFile.getSize());
@Override public void encode(final FlowFileRecord flowFile, final OutputStream destination) throws IOException { final DataOutputStream out = new DataOutputStream(destination); out.writeInt(flowFile.getAttributes().size()); for (final Map.Entry<String, String> entry : flowFile.getAttributes().entrySet()) { writeString(entry.getKey(), out); writeString(entry.getValue(), out); } out.writeLong(flowFile.getLineageStartDate()); out.writeLong(flowFile.getEntryDate()); }
@Override public int compare(final FlowFileRecord f1, final FlowFileRecord f2) { int returnVal = 0; final boolean f1Penalized = f1.isPenalized(); final boolean f2Penalized = f2.isPenalized(); if (f1.getPenaltyExpirationMillis() < f2.getPenaltyExpirationMillis()) { return -1; } else if (f1.getPenaltyExpirationMillis() > f2.getPenaltyExpirationMillis()) { return 1; final ContentClaim claim1 = f1.getContentClaim(); final ContentClaim claim2 = f2.getContentClaim(); final int claimOffsetComparison = Long.compare(f1.getContentClaimOffset(), f2.getContentClaimOffset()); if (claimOffsetComparison != 0) { return claimOffsetComparison; return Long.compare(f1.getId(), f2.getId());
/** * Creates a FlowFileDTO from the specified FlowFileRecord. * * @param record record * @return dto */ public FlowFileDTO createFlowFileDTO(final FlowFileRecord record) { final Date now = new Date(); final FlowFileDTO dto = new FlowFileDTO(); dto.setUuid(record.getAttribute(CoreAttributes.UUID.key())); dto.setFilename(record.getAttribute(CoreAttributes.FILENAME.key())); dto.setPenalized(record.isPenalized()); dto.setSize(record.getSize()); dto.setAttributes(record.getAttributes()); final long queuedDuration = now.getTime() - record.getLastQueueDate(); dto.setQueuedDuration(queuedDuration); final long age = now.getTime() - record.getLineageStartDate(); dto.setLineageDuration(age); final ContentClaim contentClaim = record.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); dto.setContentClaimSection(resourceClaim.getSection()); dto.setContentClaimContainer(resourceClaim.getContainer()); dto.setContentClaimIdentifier(resourceClaim.getId()); dto.setContentClaimOffset(contentClaim.getOffset() + record.getContentClaimOffset()); dto.setContentClaimFileSizeBytes(record.getSize()); dto.setContentClaimFileSize(FormatUtils.formatDataSize(record.getSize())); } return dto; }
final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim == null) { resourceClaim = null; stream = new ByteArrayInputStream(new byte[0]); } else { resourceClaim = flowFile.getContentClaim().getResourceClaim(); stream = contentRepository.read(flowFile.getContentClaim()); final long contentClaimOffset = flowFile.getContentClaimOffset(); if (contentClaimOffset > 0L) { StreamUtils.skip(stream, contentClaimOffset); stream = new LimitingInputStream(stream, flowFile.getSize()); .setFlowFileUUID(flowFile.getAttribute(CoreAttributes.UUID.key())) .setAttributes(flowFile.getAttributes(), Collections.emptyMap()) .setTransitUri(requestUri) .setEventTime(System.currentTimeMillis()) .setFlowFileEntryDate(flowFile.getEntryDate()) .setLineageStartDate(flowFile.getLineageStartDate()) .setComponentType(flowManager.getRootGroup().getName()) .setComponentId(flowManager.getRootGroupId()) contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize());
private ProvenanceEventRecord createDropProvenanceEvent(final FlowFileRecord flowFile, final String requestor) { final ProvenanceEventBuilder builder = provRepository.eventBuilder(); builder.fromFlowFile(flowFile); builder.setEventType(ProvenanceEventType.DROP); builder.setLineageStartDate(flowFile.getLineageStartDate()); builder.setComponentId(getIdentifier()); builder.setComponentType("Connection"); builder.setAttributes(flowFile.getAttributes(), Collections.emptyMap()); builder.setDetails("FlowFile Queue emptied by " + requestor); builder.setSourceQueueIdentifier(getIdentifier()); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setPreviousContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset(), flowFile.getSize()); } return builder.build(); }
.setEventTime(System.currentTimeMillis()); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); provenanceEvents.add(provenanceEvent); final long flowFileLife = System.currentTimeMillis() - flowFile.getEntryDate(); logger.info("{} terminated due to FlowFile expiration; life of FlowFile = {} ms", new Object[] {flowFile, flowFileLife});
private ProvenanceEventRecord createSendEvent(final FlowFileRecord flowFile, final NodeIdentifier nodeIdentifier) { final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder() .fromFlowFile(flowFile) .setEventType(ProvenanceEventType.SEND) .setDetails("Re-distributed for Load-balanced connection") .setComponentId(flowFileQueue.getIdentifier()) .setComponentType("Connection") .setSourceQueueIdentifier(flowFileQueue.getIdentifier()) .setSourceSystemFlowFileIdentifier(flowFile.getAttribute(CoreAttributes.UUID.key())) .setTransitUri("nifi://" + nodeIdentifier.getApiAddress() + "/loadbalance/" + flowFileQueue.getIdentifier()); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setCurrentContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); builder.setPreviousContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); } final ProvenanceEventRecord sendEvent = builder.build(); return sendEvent; }
private int ensureNotTruncated(final int length) throws EOFException { if (length > -1) { bytesRead += length; return length; } if (bytesRead < flowFile.getSize()) { throw new EOFException("Expected " + flowFile + " to contain " + flowFile.getSize() + " bytes but the content repository only had " + bytesRead + " bytes for it"); } return length; } };
final List<ResourceClaim> resourceClaims = new ArrayList<>(); for (final FlowFileRecord flowFile : toSwap) { contentSize += flowFile.getSize(); if (flowFile.getId() > maxFlowFileId) { maxFlowFileId = flowFile.getId(); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim != null) { resourceClaims.add(contentClaim.getResourceClaim());
private void setType(final RepositoryRecordType newType) { if (newType == this.type) { return; } if (this.type == RepositoryRecordType.CREATE) { // Because we don't copy updated attributes to `this.updatedAttributes` for CREATE records, we need to ensure // that if a record is changed from CREATE to anything else that we do properly update the `this.updatedAttributes` field. this.updatedAttributes = new HashMap<>(getCurrent().getAttributes()); } this.type = newType; }
@Override public FlowFile clone(FlowFile example, final long offset, final long size) { verifyTaskActive(); example = validateRecordState(example); final StandardRepositoryRecord exampleRepoRecord = getRecord(example); final FlowFileRecord currRec = exampleRepoRecord.getCurrent(); final ContentClaim claim = exampleRepoRecord.getCurrentClaim(); if (offset + size > example.getSize()) { throw new FlowFileHandlingException("Specified offset of " + offset + " and size " + size + " exceeds size of " + example.toString()); } final StandardFlowFileRecord.Builder builder = new StandardFlowFileRecord.Builder().fromFlowFile(currRec); builder.id(context.getNextFlowFileSequence()); builder.contentClaimOffset(currRec.getContentClaimOffset() + offset); builder.size(size); final String newUuid = UUID.randomUUID().toString(); builder.addAttribute(CoreAttributes.UUID.key(), newUuid); final FlowFileRecord clone = builder.build(); if (claim != null) { context.getContentRepository().incrementClaimaintCount(claim); } final StandardRepositoryRecord record = new StandardRepositoryRecord(null); record.setWorking(clone, clone.getAttributes()); records.put(clone.getId(), record); if (offset == 0L && size == example.getSize()) { provenanceReporter.clone(example, clone); } else { registerForkEvent(example, clone); } return clone; }
final FlowFileRecord eventFlowFile = flowFileRecordMap.get(rawEvent.getFlowFileUuid()); if (eventFlowFile != null) { final StandardRepositoryRecord repoRecord = records.get(eventFlowFile.getId()); final long size = eventFlowFile.getSize(); final long originalOffset = repoRecord.getOriginal().getContentClaimOffset(); final long originalSize = repoRecord.getOriginal().getSize();
public FlowFileRecordFieldMap(final FlowFileRecord flowFile, final RecordSchema schema) { this.flowFile = flowFile; this.schema = schema; final RecordField contentClaimField = schema.getField(FlowFileSchema.CONTENT_CLAIM); contentClaimSchema = new RecordSchema(contentClaimField.getSubFields()); contentClaim = flowFile.getContentClaim() == null ? null : new ContentClaimFieldMap(flowFile.getContentClaim(), flowFile.getContentClaimOffset(), contentClaimSchema); }
@Override public ContentClaim getOriginalClaim() { return (originalFlowFileRecord == null) ? null : originalFlowFileRecord.getContentClaim(); }
private void updateEventContentClaims(final ProvenanceEventBuilder builder, final FlowFile flowFile, final StandardRepositoryRecord repoRecord) { final ContentClaim originalClaim = repoRecord.getOriginalClaim(); if (originalClaim == null) { builder.setCurrentContentClaim(null, null, null, null, 0L); } else { final ResourceClaim resourceClaim = originalClaim.getResourceClaim(); builder.setCurrentContentClaim( resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), repoRecord.getOriginal().getContentClaimOffset() + originalClaim.getOffset(), repoRecord.getOriginal().getSize()); builder.setPreviousContentClaim( resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), repoRecord.getOriginal().getContentClaimOffset() + originalClaim.getOffset(), repoRecord.getOriginal().getSize()); } }
protected QueueSize drop(final List<FlowFileRecord> flowFiles, final String requestor) throws IOException { // Create a Provenance Event and a FlowFile Repository record for each FlowFile final List<ProvenanceEventRecord> provenanceEvents = new ArrayList<>(flowFiles.size()); final List<RepositoryRecord> flowFileRepoRecords = new ArrayList<>(flowFiles.size()); for (final FlowFileRecord flowFile : flowFiles) { provenanceEvents.add(createDropProvenanceEvent(flowFile, requestor)); flowFileRepoRecords.add(createDeleteRepositoryRecord(flowFile)); } long dropContentSize = 0L; for (final FlowFileRecord flowFile : flowFiles) { dropContentSize += flowFile.getSize(); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim == null) { continue; } final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); if (resourceClaim == null) { continue; } resourceClaimManager.decrementClaimantCount(resourceClaim); } provRepository.registerEvents(provenanceEvents); flowFileRepository.updateRepository(flowFileRepoRecords); return new QueueSize(flowFiles.size(), dropContentSize); }
private long drainQueue(final Queue<FlowFileRecord> sourceQueue, final List<FlowFileRecord> destination, int maxResults, final Set<FlowFileRecord> expiredRecords, final long expirationMillis) { long drainedSize = 0L; FlowFileRecord pulled; while (destination.size() < maxResults && (pulled = sourceQueue.poll()) != null) { if (isExpired(pulled, expirationMillis)) { expiredRecords.add(pulled); if (expiredRecords.size() >= MAX_EXPIRED_RECORDS_PER_ITERATION) { break; } } else { if (pulled.isPenalized()) { sourceQueue.add(pulled); break; } destination.add(pulled); } drainedSize += pulled.getSize(); } return drainedSize; }
recordIdMap.put(flowFile.getAttribute(CoreAttributes.UUID.key()), flowFile); expiredRecords.add(record); expiredReporter.expire(flowFile, "Expiration Threshold = " + connection.getFlowFileQueue().getFlowFileExpiration()); decrementClaimCount(flowFile.getContentClaim()); final long flowFileLife = System.currentTimeMillis() - flowFile.getEntryDate(); final Object terminator = connectable instanceof ProcessorNode ? ((ProcessorNode) connectable).getProcessor() : connectable; LOG.info("{} terminated by {} due to FlowFile expiration; life of FlowFile = {} ms", new Object[] {flowFile, terminator, flowFileLife});