private AsyncLineageSubmission submitLineageComputation(final Collection<String> flowFileUuids, final NiFiUser user, final LineageComputationType computationType, final Long eventId, final long startTimestamp, final long endTimestamp) { final List<File> indexDirs = indexConfig.getIndexDirectories(startTimestamp, endTimestamp); final AsyncLineageSubmission result = new AsyncLineageSubmission(computationType, eventId, flowFileUuids, indexDirs.size(), user == null ? null : user.getIdentity()); lineageSubmissionMap.put(result.getLineageIdentifier(), result); for (final File indexDir : indexDirs) { queryExecService.submit(new ComputeLineageRunnable(flowFileUuids, user, result, indexDir)); } return result; }
private AsyncLineageSubmission submitLineageComputation(final Collection<String> flowFileUuids, final NiFiUser user, final LineageComputationType computationType, final Long eventId) { final String userId = user == null ? null : user.getIdentity(); final AsyncLineageSubmission result = new AsyncLineageSubmission(computationType, eventId, flowFileUuids, 1, userId); lineageSubmissionMap.put(result.getLineageIdentifier(), result); final Filter<ProvenanceEventRecord> filter = new Filter<ProvenanceEventRecord>() { @Override public boolean select(final ProvenanceEventRecord event) { if (!isAuthorized(event, user)) { return false; } if (flowFileUuids.contains(event.getFlowFileUuid())) { return true; } for (final String parentId : event.getParentUuids()) { if (flowFileUuids.contains(parentId)) { return true; } } for (final String childId : event.getChildUuids()) { if (flowFileUuids.contains(childId)) { return true; } } return false; } }; queryExecService.submit(new ComputeLineageRunnable(ringBuffer, filter, result)); return result; }
@Override public ComputeLineageSubmission submitLineageComputation(final long eventId, final NiFiUser user) { final ProvenanceEventRecord event = getEvent(eventId); if (event == null) { final String userId = user == null ? null : user.getIdentity(); final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, userId); result.getResult().setError("Could not find event with ID " + eventId); lineageSubmissionMap.put(result.getLineageIdentifier(), result); return result; } return submitLineageComputation(Collections.singleton(event.getFlowFileUuid()), user, LineageComputationType.FLOWFILE_LINEAGE, eventId); }
private ComputeLineageSubmission submitLineageComputation(final Collection<String> flowFileUuids, final NiFiUser user, final EventAuthorizer eventAuthorizer, final LineageComputationType computationType, final Long eventId, final long startTimestamp, final long endTimestamp) { final List<File> indexDirs = directoryManager.getDirectories(startTimestamp, endTimestamp); final AsyncLineageSubmission submission = new AsyncLineageSubmission(computationType, eventId, flowFileUuids, indexDirs.size(), user == null ? null : user.getIdentity()); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); final BooleanQuery lineageQuery = buildLineageQuery(flowFileUuids); final List<File> indexDirectories = directoryManager.getDirectories(startTimestamp, endTimestamp); if (indexDirectories.isEmpty()) { submission.getResult().update(Collections.emptyList(), 0L); } else { Collections.sort(indexDirectories, DirectoryUtils.OLDEST_INDEX_FIRST); for (final File indexDir : indexDirectories) { queryExecutor.submit(new QueryTask(lineageQuery, submission.getResult(), MAX_LINEAGE_NODES, indexManager, indexDir, eventStore, eventAuthorizer, EventTransformer.PLACEHOLDER_TRANSFORMER)); } } // Some computations will complete very quickly. In this case, we don't want to wait // for the client to submit a second query to obtain the result. Instead, we want to just // wait some short period of time for the computation to complete before returning the submission. try { submission.getResult().awaitCompletion(500, TimeUnit.MILLISECONDS); } catch (final InterruptedException ie) { Thread.currentThread().interrupt(); } return submission; }
if (event == null) { final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.<String>emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().update(Collections.emptyList(), 0L); return submission; default: final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.<String>emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its children cannot be expanded"); return submission; lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
if (!eventOption.isPresent()) { final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().update(Collections.emptyList(), 0L); return submission; eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its children cannot be expanded"); return submission; final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Failed to expand children for lineage of event with ID " + eventId + " due to: " + e); return submission;
@Override public ComputeLineageSubmission submitExpandParents(final long eventId, final NiFiUser user) { final String userId = user == null ? null : user.getIdentity(); final ProvenanceEventRecord event = getEvent(eventId, user); if (event == null) { final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().update(Collections.emptyList(), 0L); return submission; } switch (event.getEventType()) { case JOIN: case FORK: case REPLAY: case CLONE: return submitLineageComputation(event.getParentUuids(), user, LineageComputationType.EXPAND_PARENTS, eventId); default: { final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its parents cannot be expanded"); return submission; } } }
@Override public ComputeLineageSubmission submitExpandChildren(final long eventId, final NiFiUser user) { final String userId = user == null ? null : user.getIdentity(); final ProvenanceEventRecord event = getEvent(eventId, user); if (event == null) { final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().update(Collections.emptyList(), 0L); return submission; } switch (event.getEventType()) { case JOIN: case FORK: case REPLAY: case CLONE: return submitLineageComputation(event.getChildUuids(), user, LineageComputationType.EXPAND_CHILDREN, eventId); default: { final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_CHILDREN, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its children cannot be expanded"); return submission; } } }
@Override public ComputeLineageSubmission submitLineageComputation(final long eventId, final NiFiUser user, final EventAuthorizer eventAuthorizer) { final Optional<ProvenanceEventRecord> eventOption; try { eventOption = eventStore.getEvent(eventId); } catch (final Exception e) { logger.error("Failed to retrieve Provenance Event with ID " + eventId + " to calculate data lineage due to: " + e, e); final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, user == null ? null : user.getIdentity()); result.getResult().setError("Failed to retrieve Provenance Event with ID " + eventId + ". See logs for more information."); return result; } if (!eventOption.isPresent()) { final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.emptySet(), 1, user == null ? null : user.getIdentity()); result.getResult().setError("Could not find Provenance Event with ID " + eventId); lineageSubmissionMap.put(result.getLineageIdentifier(), result); return result; } final ProvenanceEventRecord event = eventOption.get(); return submitLineageComputation(Collections.singleton(event.getFlowFileUuid()), user, eventAuthorizer, LineageComputationType.FLOWFILE_LINEAGE, eventId, event.getLineageStartDate(), Long.MAX_VALUE); }
if (!eventOption.isPresent()) { final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().update(Collections.emptyList(), 0L); return submission; eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its parents cannot be expanded"); return submission; final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
if (event == null) { final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().update(Collections.emptyList(), 0L); return submission; default: { final AsyncLineageSubmission submission = new AsyncLineageSubmission(LineageComputationType.EXPAND_PARENTS, eventId, Collections.<String>emptyList(), 1, userId); lineageSubmissionMap.put(submission.getLineageIdentifier(), submission); submission.getResult().setError("Event ID " + eventId + " indicates an event of type " + event.getEventType() + " so its parents cannot be expanded"); return submission; lineageSubmissionMap.put(submission.getLineageIdentifier(), submission);
@Override public ComputeLineageSubmission submitLineageComputation(final long eventId, final NiFiUser user) { final ProvenanceEventRecord event; try { event = getEvent(eventId); } catch (final Exception e) { logger.error("Failed to retrieve Provenance Event with ID " + eventId + " to calculate data lineage due to: " + e, e); final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.<String> emptySet(), 1, user == null ? null : user.getIdentity()); result.getResult().setError("Failed to retrieve Provenance Event with ID " + eventId + ". See logs for more information."); return result; } if (event == null) { final AsyncLineageSubmission result = new AsyncLineageSubmission(LineageComputationType.FLOWFILE_LINEAGE, eventId, Collections.<String> emptySet(), 1, user == null ? null : user.getIdentity()); result.getResult().setError("Could not find Provenance Event with ID " + eventId); lineageSubmissionMap.put(result.getLineageIdentifier(), result); return result; } return submitLineageComputation(Collections.singleton(event.getFlowFileUuid()), user, LineageComputationType.FLOWFILE_LINEAGE, eventId, event.getLineageStartDate(), Long.MAX_VALUE); }