private Path getPath(final ResourceClaim resourceClaim) { final Path containerPath = containers.get(resourceClaim.getContainer()); if (containerPath == null) { return null; } return containerPath.resolve(resourceClaim.getSection()).resolve(resourceClaim.getId()); }
@Override public Object getFieldValue(final String fieldName) { switch (fieldName) { case ContentClaimSchema.CLAIM_CONTAINER: return resourceClaim.getContainer(); case ContentClaimSchema.CLAIM_SECTION: return resourceClaim.getSection(); case ContentClaimSchema.CLAIM_IDENTIFIER: return resourceClaim.getId(); case ContentClaimSchema.LOSS_TOLERANT: return resourceClaim.isLossTolerant(); } return null; }
@Override public int compareTo(final ContentClaim o) { final int resourceComp = resourceClaim.compareTo(o.getResourceClaim()); if (resourceComp != 0) { return resourceComp; } return Long.compare(offset, o.getOffset()); }
final String container = claim.getContainer(); final BlockingQueue<ResourceClaim> claimQueue = reclaimable.get(container);
private boolean isDestructable(final ContentClaim claim) { if (claim == null) { return false; } final ResourceClaim resourceClaim = claim.getResourceClaim(); if (resourceClaim == null) { return false; } return !resourceClaim.isInUse(); }
private boolean remove(final ResourceClaim claim) { if (claim == null) { return false; } // If the claim is still in use, we won't remove it. if (claim.isInUse()) { return false; } Path path = null; try { path = getPath(claim); } catch (final ContentNotFoundException cnfe) { } // Ensure that we have no writable claim streams for this resource claim final ByteCountingOutputStream bcos = writableClaimStreams.remove(claim); if (bcos != null) { try { bcos.close(); } catch (final IOException e) { LOG.warn("Failed to close Output Stream for {} due to {}", claim, e); } } final File file = path.toFile(); if (!file.delete() && file.exists()) { LOG.warn("Unable to delete {} at path {}", new Object[]{claim, path}); return false; } return true; }
private Path getArchivePath(final ResourceClaim claim) { final String claimId = claim.getId(); final Path containerPath = containers.get(claim.getContainer()); final Path archivePath = containerPath.resolve(claim.getSection()).resolve(ARCHIVE_DIR_NAME).resolve(claimId); return archivePath; }
private void serializeContentClaim(final ContentClaim claim, final long offset, final DataOutputStream out) throws IOException { if (claim == null) { out.write(0); } else { out.write(1); final ResourceClaim resourceClaim = claim.getResourceClaim(); writeString(resourceClaim.getId(), out); writeString(resourceClaim.getContainer(), out); writeString(resourceClaim.getSection(), out); out.writeLong(claim.getOffset()); out.writeLong(claim.getLength()); out.writeLong(offset); out.writeBoolean(resourceClaim.isLossTolerant()); } }
boolean archive(final ResourceClaim claim) throws IOException { if (!archiveData) { return false; } if (claim.isInUse()) { return false; } // If the claim count is decremented to 0 (<= 0 as a 'defensive programming' strategy), ensure that // we close the stream if there is one. There may be a stream open if create() is called and then // claimant count is removed without writing to the claim (or more specifically, without closing the // OutputStream that is returned when calling write() ). final OutputStream out = writableClaimStreams.remove(claim); if (out != null) { try { out.close(); } catch (final IOException ioe) { LOG.warn("Unable to close Output Stream for " + claim, ioe); } } final Path curPath = getPath(claim); if (curPath == null) { return false; } final boolean archived = archive(curPath); LOG.debug("Successfully moved {} to archive", claim); return archived; }
@Override public int compareTo(final ContentClaim o) { final int resourceComp = resourceClaim.compareTo(o.getResourceClaim()); if (resourceComp != 0) { return resourceComp; } return Long.compare(offset, o.getOffset()); }
/** * Provides the natural ordering for ResourceClaim objects. By default they are sorted by their id, then container, then section * * @param other other claim * @return x such that x <= -1 if this is less than other; * x=0 if this.equals(other); * x >= 1 if this is greater than other */ @Override default int compareTo(final ResourceClaim other) { final int idComparison = getId().compareTo(other.getId()); if (idComparison != 0) { return idComparison; } final int containerComparison = getContainer().compareTo(other.getContainer()); if (containerComparison != 0) { return containerComparison; } return getSection().compareTo(other.getSection()); } }
out.writeBoolean(true); final ResourceClaim resourceClaim = claim.getResourceClaim(); out.writeUTF(resourceClaim.getId()); out.writeUTF(resourceClaim.getContainer()); out.writeUTF(resourceClaim.getSection()); out.writeLong(claim.getOffset()); out.writeLong(claim.getLength()); out.writeLong(flowFile.getContentClaimOffset()); out.writeBoolean(resourceClaim.isLossTolerant());
@Override public boolean equals(final Object other) { if (this == other) { return true; } if (other == null) { return false; } if (hashCode != other.hashCode()) { // We check hash code before instanceof because instanceof is fairly expensive and for // StandardResourceClaim, calling hashCode() simply returns a pre-calculated value. return false; } if (!(other instanceof ResourceClaim)) { return false; } final ResourceClaim otherClaim = (ResourceClaim) other; return id.equals(otherClaim.getId()) && container.equals(otherClaim.getContainer()) && section.equals(otherClaim.getSection()); }
@Override public Object getFieldValue(final String fieldName) { switch (fieldName) { case ContentClaimSchema.CLAIM_CONTAINER: return resourceClaim.getContainer(); case ContentClaimSchema.CLAIM_SECTION: return resourceClaim.getSection(); case ContentClaimSchema.CLAIM_IDENTIFIER: return resourceClaim.getId(); case ContentClaimSchema.LOSS_TOLERANT: return resourceClaim.isLossTolerant(); } return null; }
private Path getPath(final ContentClaim claim, final boolean verifyExists) throws ContentNotFoundException { final ResourceClaim resourceClaim = claim.getResourceClaim(); final Path containerPath = containers.get(resourceClaim.getContainer()); if (containerPath == null) { if (verifyExists) { throw new ContentNotFoundException(claim); } else { return null; } } // Create the Path that points to the data Path resolvedPath = containerPath.resolve(resourceClaim.getSection()).resolve(resourceClaim.getId()); // If the data does not exist, create a Path that points to where the data would exist in the archive directory. if (!Files.exists(resolvedPath)) { resolvedPath = getArchivePath(claim.getResourceClaim()); if (verifyExists && !Files.exists(resolvedPath)) { throw new ContentNotFoundException(claim); } } return resolvedPath; }
private void serializeContentClaim(final ContentClaim claim, final long offset, final DataOutputStream out) throws IOException { if (claim == null) { out.write(0); } else { out.write(1); final ResourceClaim resourceClaim = claim.getResourceClaim(); writeString(resourceClaim.getId(), out); writeString(resourceClaim.getContainer(), out); writeString(resourceClaim.getSection(), out); out.writeLong(claim.getOffset()); out.writeLong(claim.getLength()); out.writeLong(offset); out.writeBoolean(resourceClaim.isLossTolerant()); } }
private void updateEventContentClaims(final ProvenanceEventBuilder builder, final FlowFile flowFile, final StandardRepositoryRecord repoRecord) { final ContentClaim originalClaim = repoRecord.getOriginalClaim(); if (originalClaim == null) { builder.setCurrentContentClaim(null, null, null, null, 0L); } else { final ResourceClaim resourceClaim = originalClaim.getResourceClaim(); builder.setCurrentContentClaim( resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), repoRecord.getOriginal().getContentClaimOffset() + originalClaim.getOffset(), repoRecord.getOriginal().getSize()); builder.setPreviousContentClaim( resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), repoRecord.getOriginal().getContentClaimOffset() + originalClaim.getOffset(), repoRecord.getOriginal().getSize()); } }
private ProvenanceEventRecord createDropEvent(final FlowFileRecord flowFile, final String details) { final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder() .fromFlowFile(flowFile) .setEventType(ProvenanceEventType.DROP) .setDetails(details) .setComponentId(flowFileQueue.getIdentifier()) .setComponentType("Connection") .setSourceQueueIdentifier(flowFileQueue.getIdentifier()); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setCurrentContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); builder.setPreviousContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); } final ProvenanceEventRecord dropEvent = builder.build(); return dropEvent; }
if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); builder.setPreviousContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize());
private void registerReceiveProvenanceEvents(final List<RemoteFlowFileRecord> flowFiles, final String nodeName, final String connectionId, final long startTimestamp) { final long duration = System.currentTimeMillis() - startTimestamp; final List<ProvenanceEventRecord> events = new ArrayList<>(flowFiles.size()); for (final RemoteFlowFileRecord remoteFlowFile : flowFiles) { final FlowFileRecord flowFileRecord = remoteFlowFile.getFlowFile(); final ProvenanceEventBuilder provenanceEventBuilder = new StandardProvenanceEventRecord.Builder() .fromFlowFile(flowFileRecord) .setEventType(ProvenanceEventType.RECEIVE) .setTransitUri("nifi://" + nodeName + "/loadbalance/" + connectionId) .setSourceSystemFlowFileIdentifier(remoteFlowFile.getRemoteUuid()) .setEventDuration(duration) .setComponentId(connectionId) .setComponentType("Load Balanced Connection"); final ContentClaim contentClaim = flowFileRecord.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); provenanceEventBuilder.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset() + flowFileRecord.getContentClaimOffset(), flowFileRecord.getSize()); } final ProvenanceEventRecord provenanceEvent = provenanceEventBuilder.build(); events.add(provenanceEvent); } provenanceRepository.registerEvents(events); }