private Iterable<Relation> getProgramRelations(NamespacedEntityId data, long start, long end, Predicate<Relation> filter) { if (data instanceof DatasetId) { return lineageStoreReader.getRelations((DatasetId) data, start, end, filter); } if (data instanceof StreamId) { return lineageStoreReader.getRelations((StreamId) data, start, end, filter); } throw new IllegalStateException("Unknown data type " + data); }
Set<NamespacedEntityId> entities = lineageReader.getEntitiesForRun(run1); Assert.assertTrue(entities.isEmpty()); Tasks.waitFor(true, () -> expectedLineage.equals(lineageReader.getEntitiesForRun(run1)), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); Assert.assertTrue(lineageReader.getRelations(spark1, 0L, Long.MAX_VALUE, x -> true).isEmpty());
/** * @return metadata associated with a run */ public Set<MetadataRecord> getMetadataForRun(ProgramRunId run) { try { entityExistenceVerifier.ensureExists(run); } catch (NotFoundException e) { // metadata apis does not support not found as they cannot perform existence check for custom entity so just // return an empty set return Collections.emptySet(); } Set<MetadataEntity> runEntities = lineageStoreReader.getEntitiesForRun(run).stream() .map(NamespacedEntityId::toMetadataEntity) .collect(Collectors.toSet()); // No entities associated with the run, but run exists. if (runEntities.isEmpty()) { return ImmutableSet.of(); } RunId runId = RunIds.fromString(run.getRun()); // The entities returned by lineageStore does not contain application ApplicationId application = run.getParent().getParent(); runEntities.add(application.toMetadataEntity()); return metadataStore.getSnapshotBeforeTime(MetadataScope.USER, runEntities, RunIds.getTime(runId, TimeUnit.MILLISECONDS)); }
/** * @return metadata associated with a run */ public Set<MetadataRecordV2> getMetadataForRun(ProgramRunId run) { try { entityExistenceVerifier.ensureExists(run); } catch (NotFoundException e) { // metadata apis does not support not found as they cannot perform existence check for custom entity so just // return an empty set return Collections.emptySet(); } Set<MetadataEntity> runEntities = lineageStoreReader.getEntitiesForRun(run).stream() .map(NamespacedEntityId::toMetadataEntity) .collect(Collectors.toSet()); // No entities associated with the run, but run exists. if (runEntities.isEmpty()) { return ImmutableSet.of(); } RunId runId = RunIds.fromString(run.getRun()); // The entities returned by lineageStore does not contain application ApplicationId application = run.getParent().getParent(); runEntities.add(application.toMetadataEntity()); return metadataStore.getSnapshotBeforeTime(MetadataScope.USER, runEntities, RunIds.getTime(runId, TimeUnit.MILLISECONDS)); }
private Iterable<Relation> getProgramRelations(NamespacedEntityId data, long start, long end, Predicate<Relation> filter) { if (!(data instanceof DatasetId)) { throw new IllegalStateException("Unknown data type " + data); } return lineageStoreReader.getRelations((DatasetId) data, start, end, filter); }
LOG.trace("Visiting program {}", p); Iterable<Relation> datasetRelations = lineageStoreReader.getRelations(p, scanRange.getStart(), scanRange.getEnd(), scanRange.getFilter());
LOG.trace("Visiting program {}", p); Iterable<Relation> datasetRelations = lineageStoreReader.getRelations(p, scanRange.getStart(), scanRange.getEnd(), scanRange.getFilter());