private void registerDequeuedRecord(final FlowFileRecord flowFile, final Connection connection) { final StandardRepositoryRecord record = new StandardRepositoryRecord(connection.getFlowFileQueue(), flowFile); records.put(flowFile.getId(), record); flowFilesIn++; contentSizeIn += flowFile.getSize(); Set<FlowFileRecord> set = unacknowledgedFlowFiles.get(connection.getFlowFileQueue()); if (set == null) { set = new HashSet<>(); unacknowledgedFlowFiles.put(connection.getFlowFileQueue(), set); } set.add(flowFile); incrementConnectionOutputCounts(connection, flowFile); }
@Override public Collection<FlowFileQueue> getAllQueues() { final Collection<Connection> connections = flowController.getFlowManager().findAllConnections(); final List<FlowFileQueue> queues = new ArrayList<>(connections.size()); for (final Connection connection : connections) { queues.add(connection.getFlowFileQueue()); } return queues; } }
public static boolean flowFilesQueued(final Connectable connectable) { for (final Connection conn : connectable.getIncomingConnections()) { if (!conn.getFlowFileQueue().isActiveQueueEmpty()) { return true; } } return false; }
@Override public void verifyList(String id) { final Connection connection = locateConnection(id); final FlowFileQueue queue = connection.getFlowFileQueue(); verifyList(queue); }
@Override public DropFlowFileStatus getFlowFileDropRequest(String connectionId, String dropRequestId) { final Connection connection = locateConnection(connectionId); final FlowFileQueue queue = connection.getFlowFileQueue(); final DropFlowFileStatus dropRequest = queue.getDropFlowFileStatus(dropRequestId); if (dropRequest == null) { throw new ResourceNotFoundException(String.format("Unable to find drop request with id '%s'.", dropRequestId)); } return dropRequest; }
@Override public DropFlowFileStatus createFlowFileDropRequest(String id, String dropRequestId) { final Connection connection = locateConnection(id); final FlowFileQueue queue = connection.getFlowFileQueue(); final NiFiUser user = NiFiUserUtils.getNiFiUser(); if (user == null) { throw new WebApplicationException(new Throwable("Unable to access details for current user.")); } return queue.dropFlowFiles(dropRequestId, user.getIdentity()); }
@Override public ListFlowFileStatus getFlowFileListingRequest(String connectionId, String listingRequestId) { final Connection connection = locateConnection(connectionId); final FlowFileQueue queue = connection.getFlowFileQueue(); final ListFlowFileStatus listRequest = queue.getListFlowFileStatus(listingRequestId); if (listRequest == null) { throw new ResourceNotFoundException(String.format("Unable to find listing request with id '%s'.", listingRequestId)); } return listRequest; }
@Override public ListFlowFileStatus deleteFlowFileListingRequest(String connectionId, String listingRequestId) { final Connection connection = locateConnection(connectionId); final FlowFileQueue queue = connection.getFlowFileQueue(); final ListFlowFileStatus listFlowFileStatus = queue.cancelListFlowFileRequest(listingRequestId); if (listFlowFileStatus == null) { throw new ResourceNotFoundException(String.format("Unable to find listing request with id '%s'.", listingRequestId)); } return listFlowFileStatus; }
public void onConnectionAdded(final Connection connection) { allConnections.put(connection.getIdentifier(), connection); if (flowController.isInitialized()) { connection.getFlowFileQueue().startLoadBalancing(); } }
@Override public DropFlowFileStatus deleteFlowFileDropRequest(String connectionId, String dropRequestId) { final Connection connection = locateConnection(connectionId); final FlowFileQueue queue = connection.getFlowFileQueue(); final DropFlowFileStatus dropFlowFileStatus = queue.cancelDropFlowFileRequest(dropRequestId); if (dropFlowFileStatus == null) { throw new ResourceNotFoundException(String.format("Unable to find drop request with id '%s'.", dropRequestId)); } return dropFlowFileStatus; }
public void expireFlowFiles() { final Set<FlowFileRecord> expired = new HashSet<>(); final FlowFileFilter filter = new FlowFileFilter() { @Override public FlowFileFilterResult filter(final FlowFile flowFile) { return FlowFileFilterResult.REJECT_AND_CONTINUE; } }; for (final Connection conn : context.getConnectable().getIncomingConnections()) { do { expired.clear(); conn.getFlowFileQueue().poll(filter, expired); removeExpired(expired, conn); } while (!expired.isEmpty()); } }
@Override public Set<Relationship> getAvailableRelationships() { for (final Connection connection : connectable.getConnections()) { if (connection.getFlowFileQueue().isFull()) { return Collections.emptySet(); } } final Collection<Relationship> relationships = connectable.getRelationships(); if (relationships instanceof Set) { return (Set<Relationship>) relationships; } return new HashSet<>(connectable.getRelationships()); }
private int getWorkCount(final Connectable connectable) { int sum = 0; for (final Connection connection : connectable.getIncomingConnections()) { sum += connection.getFlowFileQueue().size().getObjectCount(); } return sum; }
@Override public ListFlowFileStatus createFlowFileListingRequest(String id, String listingRequestId) { final Connection connection = locateConnection(id); final FlowFileQueue queue = connection.getFlowFileQueue(); // ensure we can list verifyList(queue); return queue.listFlowFiles(listingRequestId, 100); }
public static boolean anyRelationshipAvailable(final Connectable connectable) { for (final Relationship relationship : connectable.getRelationships()) { final Collection<Connection> connections = connectable.getConnections(relationship); boolean available = true; for (final Connection connection : connections) { if (connection.getFlowFileQueue().isFull()) { available = false; break; } } if (available) { return true; } } return false; }
@Override public FlowFileRecord getFlowFile(String id, String flowFileUuid) { try { final Connection connection = locateConnection(id); final FlowFileQueue queue = connection.getFlowFileQueue(); final FlowFileRecord flowFile = queue.getFlowFile(flowFileUuid); if (flowFile == null) { throw new ResourceNotFoundException(String.format("The FlowFile with UUID %s is no longer in the active queue.", flowFileUuid)); } // get the attributes and ensure appropriate access final Map<String, String> attributes = flowFile.getAttributes(); final Authorizable dataAuthorizable = new DataAuthorizable(connection.getSourceAuthorizable()); dataAuthorizable.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser(), attributes); return flowFile; } catch (final IOException ioe) { logger.error(String.format("Unable to get the flowfile (%s) at this time.", flowFileUuid), ioe); throw new IllegalStateException("Unable to get the FlowFile at this time."); } }
public boolean isAnyRelationshipAvailable() { for (final Relationship relationship : getConnectable().getRelationships()) { final Collection<Connection> connections = getConnections(relationship); boolean available = true; for (final Connection connection : connections) { if (connection.getFlowFileQueue().isFull()) { available = false; break; } } if (available) { return true; } } return false; }
@Override public QueueSize getQueueSize() { verifyTaskActive(); int flowFileCount = 0; long byteCount = 0L; for (final Connection conn : context.getPollableConnections()) { final QueueSize queueSize = conn.getFlowFileQueue().size(); flowFileCount += queueSize.getObjectCount(); byteCount += queueSize.getByteCount(); } return new QueueSize(flowFileCount, byteCount); }
private void expireFlowFiles(final Connectable connectable) { // determine if the incoming connections for this Connectable have Expiration configured. boolean expirationConfigured = false; for (final Connection incomingConn : connectable.getIncomingConnections()) { if (FormatUtils.getTimeDuration(incomingConn.getFlowFileQueue().getFlowFileExpiration(), TimeUnit.MILLISECONDS) > 0) { expirationConfigured = true; break; } } // If expiration is not configured... don't bother running through the FlowFileQueue if (!expirationConfigured) { return; } final StandardProcessSession session = createSession(connectable); session.expireFlowFiles(); session.commit(); }
public QueueSize getTotalFlowFileCount(final ProcessGroup group) { int count = 0; long contentSize = 0L; for (final Connection connection : group.getConnections()) { final QueueSize size = connection.getFlowFileQueue().size(); count += size.getObjectCount(); contentSize += size.getByteCount(); } for (final ProcessGroup childGroup : group.getProcessGroups()) { final QueueSize size = getTotalFlowFileCount(childGroup); count += size.getObjectCount(); contentSize += size.getByteCount(); } return new QueueSize(count, contentSize); }