/** * Gets an instance of {@link LineageDataset}. The dataset instance will be created if it is not yet exist. * * @param datasetContext the {@link DatasetContext} for getting the dataset instance. * @param datasetFramework the {@link DatasetFramework} for creating the dataset instance if missing * @return an instance of {@link LineageDataset} */ public static LineageDataset getLineageDataset(DatasetContext datasetContext, DatasetFramework datasetFramework) { return getLineageDataset(datasetContext, datasetFramework, LINEAGE_DATASET_ID); }
/** * Gets an instance of {@link LineageDataset}. The dataset instance will be created if it is not yet exist. * * @param datasetContext the {@link DatasetContext} for getting the dataset instance. * @param datasetFramework the {@link DatasetFramework} for creating the dataset instance if missing * @return an instance of {@link LineageDataset} */ public static LineageDataset getLineageDataset(DatasetContext datasetContext, DatasetFramework datasetFramework) { return getLineageDataset(datasetContext, datasetFramework, LINEAGE_DATASET_ID); }
DataAccessLineageProcessor(DatasetContext datasetContext) { this.lineageDataset = LineageDataset.getLineageDataset(datasetContext, datasetFramework, lineageDatasetId); }
DataAccessLineageProcessor(DatasetContext datasetContext) { this.lineageDataset = LineageDataset.getLineageDataset(datasetContext, datasetFramework, lineageDatasetId); }
@Override public void addAccess(ProgramRunId run, StreamId streamId, AccessType accessType, @Nullable NamespacedEntityId component) { long accessTime = System.currentTimeMillis(); LOG.trace("Writing access for run {}, stream {}, accessType {}, component {}, accessTime = {}", run, streamId, accessType, component, accessTime); Transactionals.execute(transactional, context -> { LineageDataset .getLineageDataset(context, datasetFramework, getLineageDatasetId()) .addAccess(run, streamId, accessType, accessTime, component); }); }
private <T> T execute(TransactionExecutor.Function<LineageDataset, T> func) { return Transactionals.execute(transactional, context -> { LineageDataset lineageDataset = LineageDataset.getLineageDataset(context, datasetFramework, lineageDatasetId); return func.apply(lineageDataset); }); } }
private <T> T execute(TransactionExecutor.Function<LineageDataset, T> func) { return Transactionals.execute(transactional, context -> { LineageDataset lineageDataset = LineageDataset.getLineageDataset(context, datasetFramework, lineageDatasetId); return func.apply(lineageDataset); }); } }
@Override public void addAccess(ProgramRunId run, DatasetId datasetId, AccessType accessType, @Nullable NamespacedEntityId component) { // Don't record lineage for the lineage dataset itself, otherwise there would be infinite recursion if (getLineageDatasetId().equals(datasetId)) { return; } long accessTime = System.currentTimeMillis(); LOG.trace("Writing access for run {}, dataset {}, accessType {}, component {}, accessTime = {}", run, datasetId, accessType, component, accessTime); Transactionals.execute(transactional, context -> { LineageDataset .getLineageDataset(context, datasetFramework, getLineageDatasetId()) .addAccess(run, datasetId, accessType, accessTime, component); }); }
@Override public void addAccess(ProgramRunId run, DatasetId datasetId, AccessType accessType, @Nullable NamespacedEntityId component) { // Don't record lineage for the lineage dataset itself, otherwise there would be infinite recursion if (getLineageDatasetId().equals(datasetId)) { return; } long accessTime = System.currentTimeMillis(); LOG.trace("Writing access for run {}, dataset {}, accessType {}, component {}, accessTime = {}", run, datasetId, accessType, component, accessTime); Transactionals.execute(transactional, context -> { LineageDataset .getLineageDataset(context, datasetFramework, getLineageDatasetId()) .addAccess(run, datasetId, accessType, accessTime, component); }); }