private static CollapsedRelation toCollapsedRelation(NamespacedEntityId data, ProgramId program, Set<AccessType> accesses, Set<RunId> runs, Set<NamespacedEntityId> components) { Preconditions.checkState(data instanceof DatasetId || data instanceof StreamId, "%s should be an instance of dataset or stream", data); if (data instanceof DatasetId) { return new CollapsedRelation((DatasetId) data, program, accesses, runs, components); } return new CollapsedRelation((StreamId) data, program, accesses, runs, components); }
public static LineageRecord toLineageRecord(long start, long end, Lineage lineage, Set<CollapseType> collapseTypes) { Set<RelationRecord> relationBuilder = new HashSet<>(); Map<String, ProgramRecord> programBuilder = new HashMap<>(); Map<String, DataRecord> dataBuilder = new HashMap<>(); Set<CollapsedRelation> collapsedRelations = LineageCollapser.collapseRelations(lineage.getRelations(), collapseTypes); for (CollapsedRelation relation : collapsedRelations) { String dataKey = makeDataKey(relation.getData()); String programKey = makeProgramKey(relation.getProgram()); RelationRecord relationRecord = new RelationRecord(dataKey, programKey, convertAccessType(relation.getAccess()), convertRuns(relation.getRuns()), convertComponents(relation.getComponents())); relationBuilder.add(relationRecord); programBuilder.put(programKey, new ProgramRecord(relation.getProgram())); dataBuilder.put(dataKey, new DataRecord(relation.getData())); } return new LineageRecord(start, end, relationBuilder, programBuilder, dataBuilder); }
public static LineageRecord toLineageRecord(long start, long end, Lineage lineage, Set<CollapseType> collapseTypes) { Set<RelationRecord> relationBuilder = new HashSet<>(); Map<String, ProgramRecord> programBuilder = new HashMap<>(); Map<String, DataRecord> dataBuilder = new HashMap<>(); Set<CollapsedRelation> collapsedRelations = LineageCollapser.collapseRelations(lineage.getRelations(), collapseTypes); for (CollapsedRelation relation : collapsedRelations) { String dataKey = makeDataKey(relation.getData()); String programKey = makeProgramKey(relation.getProgram()); RelationRecord relationRecord = new RelationRecord(dataKey, programKey, convertAccessType(relation.getAccess()), convertRuns(relation.getRuns()), convertComponents(relation.getComponents())); relationBuilder.add(relationRecord); programBuilder.put(programKey, new ProgramRecord(relation.getProgram())); dataBuilder.put(dataKey, new DataRecord(relation.getData())); } return new LineageRecord(start, end, relationBuilder, programBuilder, dataBuilder); }
private static CollapsedRelation toCollapsedRelation(NamespacedEntityId data, ProgramId program, Set<AccessType> accesses, Set<RunId> runs, Set<NamespacedEntityId> components) { Preconditions.checkState(data instanceof DatasetId, "%s should be an instance of dataset", data); return new CollapsedRelation((DatasetId) data, program, accesses, runs, components); }
@Test public void testCollapseComponent() { Set<Relation> relations = ImmutableSet.of( new Relation(data1, service1, AccessType.READ, runId1), new Relation(data1, service1, AccessType.WRITE, runId1), new Relation(data1, service1, AccessType.READ, runId1) ); // Collapse on component Assert.assertEquals( toSet( new CollapsedRelation(data1, service1, toSet(AccessType.READ), toSet(runId1), Collections.emptySet()), new CollapsedRelation(data1, service1, toSet(AccessType.WRITE), toSet(runId1), Collections.emptySet()) ), LineageCollapser.collapseRelations(relations, ImmutableSet.of(CollapseType.COMPONENT)) ); }
@Test public void testCollapseRun() { Set<Relation> relations = ImmutableSet.of( new Relation(data1, service1, AccessType.READ, runId1), new Relation(data1, service1, AccessType.WRITE, runId1), new Relation(data1, service1, AccessType.READ, runId2) ); // Collapse on run Assert.assertEquals( toSet( new CollapsedRelation(data1, service1, toSet(AccessType.READ), toSet(runId1, runId2), Collections.emptySet()), new CollapsedRelation(data1, service1, toSet(AccessType.WRITE), toSet(runId1), Collections.emptySet()) ), LineageCollapser.collapseRelations(relations, ImmutableSet.of(CollapseType.RUN)) ); }
@Test public void testCollapseMulti() { Set<Relation> relations = ImmutableSet.of( new Relation(data1, service1, AccessType.READ, runId1), new Relation(data1, service1, AccessType.WRITE, runId1), new Relation(data1, service1, AccessType.READ, runId1), new Relation(data1, service2, AccessType.READ, runId1), new Relation(data1, service2, AccessType.READ, runId1), new Relation(data2, service1, AccessType.READ, runId1), new Relation(data2, service1, AccessType.READ, runId1) ); // Collapse on access Assert.assertEquals( toSet( new CollapsedRelation(data1, service1, toSet(AccessType.READ, AccessType.WRITE), toSet(runId1), Collections.emptySet()), new CollapsedRelation(data1, service2, toSet(AccessType.READ), toSet(runId1), Collections.emptySet()), new CollapsedRelation(data2, service1, toSet(AccessType.READ), toSet(runId1), Collections.emptySet()) ), LineageCollapser.collapseRelations(relations, ImmutableSet.of(CollapseType.ACCESS)) ); }
@Test public void testCollapseAccess() { Set<Relation> relations = ImmutableSet.of( new Relation(data1, service1, AccessType.READ, runId1), new Relation(data1, service1, AccessType.WRITE, runId1), new Relation(data1, service1, AccessType.READ, runId1) ); // Collapse on access Assert.assertEquals( toSet( new CollapsedRelation(data1, service1, toSet(AccessType.READ, AccessType.WRITE), toSet(runId1), Collections.emptySet()) ), LineageCollapser.collapseRelations(relations, ImmutableSet.of(CollapseType.ACCESS)) ); }
new CollapsedRelation(data1, service1, toSet(AccessType.READ, AccessType.WRITE, AccessType.UNKNOWN), toSet(runId1, runId2, runId3), Collections.emptySet()) ), new CollapsedRelation(data1, service1, toSet(AccessType.READ, AccessType.WRITE), toSet(runId1), Collections.emptySet()), new CollapsedRelation(data1, service1, toSet(AccessType.READ, AccessType.WRITE, AccessType.UNKNOWN), toSet(runId2), Collections.emptySet()), new CollapsedRelation(data1, service1, toSet(AccessType.READ, AccessType.UNKNOWN), toSet(runId3), Collections.emptySet()) ), new CollapsedRelation(data1, service1, toSet(AccessType.READ), toSet(runId1, runId2, runId3), Collections.emptySet()), new CollapsedRelation(data1, service1, toSet(AccessType.WRITE), toSet(runId1, runId2), Collections.emptySet()), new CollapsedRelation(data1, service1, toSet(AccessType.UNKNOWN), toSet(runId2, runId3), Collections.emptySet()) ), LineageCollapser.collapseRelations(relations, toSet(CollapseType.COMPONENT, CollapseType.RUN)) new CollapsedRelation(data1, service1, toSet(AccessType.READ, AccessType.WRITE, AccessType.UNKNOWN), toSet(runId1, runId2, runId3), Collections.emptySet())