/** * Add a program-dataset access. * * @param run program run information * @param datasetInstance dataset accessed by the program * @param accessType access type * @param accessTimeMillis time of access */ public void addAccess(ProgramRunId run, DatasetId datasetInstance, AccessType accessType, long accessTimeMillis) { addAccess(run, datasetInstance, accessType, accessTimeMillis, null); } /**
/** * Add a program-stream access. * * @param run program run information * @param stream stream accessed by the program * @param accessType access type * @param accessTimeMillis time of access */ public void addAccess(ProgramRunId run, StreamId stream, AccessType accessType, long accessTimeMillis) { addAccess(run, stream, accessType, accessTimeMillis, null); }
/** * Add a program-dataset access. * * @param run program run information * @param datasetInstance dataset accessed by the program * @param accessType access type * @param accessTimeMillis time of access */ public void addAccess(ProgramRunId run, DatasetId datasetInstance, AccessType accessType, long accessTimeMillis) { addAccess(run, datasetInstance, accessType, accessTimeMillis, null); } /**
@Override public void apply() throws Exception { lineageDataset.addAccess(run, datasetInstance, AccessType.READ, accessTimeMillis); } });
@Override public void addAccess(ProgramRunId run, StreamId streamId, AccessType accessType, @Nullable NamespacedEntityId component) { long accessTime = System.currentTimeMillis(); LOG.trace("Writing access for run {}, stream {}, accessType {}, component {}, accessTime = {}", run, streamId, accessType, component, accessTime); Transactionals.execute(transactional, context -> { LineageDataset .getLineageDataset(context, datasetFramework, getLineageDatasetId()) .addAccess(run, streamId, accessType, accessTime, component); }); }
@Override public void addAccess(ProgramRunId run, DatasetId datasetId, AccessType accessType, @Nullable NamespacedEntityId component) { // Don't record lineage for the lineage dataset itself, otherwise there would be infinite recursion if (getLineageDatasetId().equals(datasetId)) { return; } long accessTime = System.currentTimeMillis(); LOG.trace("Writing access for run {}, dataset {}, accessType {}, component {}, accessTime = {}", run, datasetId, accessType, component, accessTime); Transactionals.execute(transactional, context -> { LineageDataset .getLineageDataset(context, datasetFramework, getLineageDatasetId()) .addAccess(run, datasetId, accessType, accessTime, component); }); }
@Override public void addAccess(ProgramRunId run, DatasetId datasetId, AccessType accessType, @Nullable NamespacedEntityId component) { // Don't record lineage for the lineage dataset itself, otherwise there would be infinite recursion if (getLineageDatasetId().equals(datasetId)) { return; } long accessTime = System.currentTimeMillis(); LOG.trace("Writing access for run {}, dataset {}, accessType {}, component {}, accessTime = {}", run, datasetId, accessType, component, accessTime); Transactionals.execute(transactional, context -> { LineageDataset .getLineageDataset(context, datasetFramework, getLineageDatasetId()) .addAccess(run, datasetId, accessType, accessTime, component); }); }
@Override public void processMessage(MetadataMessage message) { if (!(message.getEntityId() instanceof ProgramRunId)) { LOG.warn("Missing program run id from the lineage access information. Ignoring the message {}", message); return; } DataAccessLineage lineage = message.getPayload(GSON, DataAccessLineage.class); ProgramRunId programRunId = (ProgramRunId) message.getEntityId(); lineageDataset.addAccess(programRunId, lineage.getDatasetId(), lineage.getAccessType(), lineage.getAccessTime(), lineage.getComponentId()); } }
@Override public void processMessage(MetadataMessage message) { if (!(message.getEntityId() instanceof ProgramRunId)) { LOG.warn("Missing program run id from the lineage access information. Ignoring the message {}", message); return; } DataAccessLineage lineage = message.getPayload(GSON, DataAccessLineage.class); ProgramRunId programRunId = (ProgramRunId) message.getEntityId(); if (lineage.getDatasetId() != null) { lineageDataset.addAccess(programRunId, lineage.getDatasetId(), lineage.getAccessType(), lineage.getAccessTime(), lineage.getComponentId()); } else if (lineage.getStreamId() != null) { lineageDataset.addAccess(programRunId, lineage.getStreamId(), lineage.getAccessType(), lineage.getAccessTime(), lineage.getComponentId()); } else { // This shouldn't happen LOG.warn("Missing dataset id from the lineage access information. Ignoring the message {}", message); } } }
lineageDataset.addAccess(run11, datasetInstance1, AccessType.READ, run11Data1AccessTime); lineageDataset.addAccess(run22, datasetInstance2, AccessType.WRITE, run22Data2AccessTime); lineageDataset.addAccess(run23, datasetInstance2, AccessType.WRITE, run23Data2AccessTime); lineageDataset.addAccess(run34, datasetInstance2, AccessType.READ_WRITE, System.currentTimeMillis()); });