/** * @return metadata associated with a run */ public Set<MetadataRecordV2> getMetadataForRun(ProgramRunId run) { try { entityExistenceVerifier.ensureExists(run); } catch (NotFoundException e) { // metadata apis does not support not found as they cannot perform existence check for custom entity so just // return an empty set return Collections.emptySet(); } Set<MetadataEntity> runEntities = lineageStoreReader.getEntitiesForRun(run).stream() .map(NamespacedEntityId::toMetadataEntity) .collect(Collectors.toSet()); // No entities associated with the run, but run exists. if (runEntities.isEmpty()) { return ImmutableSet.of(); } RunId runId = RunIds.fromString(run.getRun()); // The entities returned by lineageStore does not contain application ApplicationId application = run.getParent().getParent(); runEntities.add(application.toMetadataEntity()); return metadataStore.getSnapshotBeforeTime(MetadataScope.USER, runEntities, RunIds.getTime(runId, TimeUnit.MILLISECONDS)); }
/** * @return metadata associated with a run */ public Set<MetadataRecord> getMetadataForRun(ProgramRunId run) { try { entityExistenceVerifier.ensureExists(run); } catch (NotFoundException e) { // metadata apis does not support not found as they cannot perform existence check for custom entity so just // return an empty set return Collections.emptySet(); } Set<MetadataEntity> runEntities = lineageStoreReader.getEntitiesForRun(run).stream() .map(NamespacedEntityId::toMetadataEntity) .collect(Collectors.toSet()); // No entities associated with the run, but run exists. if (runEntities.isEmpty()) { return ImmutableSet.of(); } RunId runId = RunIds.fromString(run.getRun()); // The entities returned by lineageStore does not contain application ApplicationId application = run.getParent().getParent(); runEntities.add(application.toMetadataEntity()); return metadataStore.getSnapshotBeforeTime(MetadataScope.USER, runEntities, RunIds.getTime(runId, TimeUnit.MILLISECONDS)); }
/** * Delete the metadata for the application and the programs. */ private void deleteAppMetadata(ApplicationId appId, ApplicationSpecification appSpec) { // Remove metadata for the Application itself. metadataStore.removeMetadata(appId.toMetadataEntity()); // Remove metadata for the programs of the Application // TODO: Need to remove this we support prefix search of metadata type. // See https://issues.cask.co/browse/CDAP-3669 for (ProgramId programId : getAllPrograms(appId, appSpec)) { metadataStore.removeMetadata(programId.toMetadataEntity()); } }
@Test public void testGetMetadataEntityFromKey() { ApplicationId expectedAppId = new ApplicationId("ns1", "app1"); MDSKey mdsValueKey = MetadataKey.createValueRowKey(expectedAppId.toMetadataEntity(), "key1"); MDSKey mdsIndexKey = MetadataKey.createIndexRowKey(expectedAppId.toMetadataEntity(), "key1", "value1"); // check that we can get MetadataEntity from value and index key MetadataEntity actualAppId = MetadataKey.extractMetadataEntityFromKey(mdsValueKey.getKey()); Assert.assertEquals(expectedAppId.toMetadataEntity(), actualAppId); actualAppId = MetadataKey.extractMetadataEntityFromKey(mdsIndexKey.getKey()); Assert.assertEquals(expectedAppId.toMetadataEntity(), actualAppId); }
@Override public void process(ApplicationWithPrograms input) throws Exception { // add system metadata for apps ApplicationId appId = input.getApplicationId(); ApplicationSpecification appSpec = input.getSpecification(); // only update creation time if this is a new app Map<String, String> properties = metadataStore.getProperties(MetadataScope.SYSTEM, appId.toMetadataEntity()); SystemMetadataWriter appSystemMetadataWriter = new AppSystemMetadataWriter(metadataStore, appId, appSpec, !properties.isEmpty()); appSystemMetadataWriter.write(); // add system metadata for programs writeProgramSystemMetadata(appId, ProgramType.FLOW, appSpec.getFlows().values()); writeProgramSystemMetadata(appId, ProgramType.MAPREDUCE, appSpec.getMapReduce().values()); writeProgramSystemMetadata(appId, ProgramType.SERVICE, appSpec.getServices().values()); writeProgramSystemMetadata(appId, ProgramType.SPARK, appSpec.getSpark().values()); writeProgramSystemMetadata(appId, ProgramType.WORKER, appSpec.getWorkers().values()); writeProgramSystemMetadata(appId, ProgramType.WORKFLOW, appSpec.getWorkflows().values()); // Emit input to the next stage emit(input); }
/** * Delete the metadata for the application and the programs. */ private void deleteAppMetadata(ApplicationId appId, ApplicationSpecification appSpec) { // Remove metadata for the Application itself. metadataPublisher.publish(NamespaceId.SYSTEM, new MetadataOperation.Drop(appId.toMetadataEntity())); // Remove metadata for the programs of the Application // TODO: Need to remove this we support prefix search of metadata type. // See https://issues.cask.co/browse/CDAP-3669 for (ProgramId programId : getAllPrograms(appId, appSpec)) { metadataPublisher.publish(NamespaceId.SYSTEM, new MetadataOperation.Drop(programId.toMetadataEntity())); } }
private void generateMetadataUpdates() { store.addTags(MetadataScope.USER, dataset.toMetadataEntity(), datasetTags); store.addProperties(MetadataScope.USER, app.toMetadataEntity(), appProperties); store.addTags(MetadataScope.USER, app.toMetadataEntity(), appTags); store.addTags(MetadataScope.USER, service.toMetadataEntity(), tags); store.removeTags(MetadataScope.USER, service.toMetadataEntity()); store.removeTags(MetadataScope.USER, dataset.toMetadataEntity(), datasetTags); store.removeMetadata(MetadataScope.USER, app.toMetadataEntity()); }
@Test public void testGetMetadataKey() { MDSKey mdsValueKey = MetadataKey.createValueRowKey(new ApplicationId("ns1", "app1").toMetadataEntity(), "key1"); MDSKey mdsIndexKey = MetadataKey.createIndexRowKey(new ApplicationId("ns1", "app1").toMetadataEntity(), "key1", "value1"); // assert that metadata key for both index and value column is key1 Assert.assertEquals("key1", MetadataKey.extractMetadataKey(mdsValueKey.getKey())); Assert.assertEquals("key1", MetadataKey.extractMetadataKey(mdsIndexKey.getKey())); }
@Test public void testGetTargetType() { MDSKey mdsValueKey = MetadataKey.createValueRowKey(new ApplicationId("ns1", "app1").toMetadataEntity(), "key1"); MDSKey mdsIndexKey = MetadataKey.createIndexRowKey(new ApplicationId("ns1", "app1").toMetadataEntity(), "key1", "value1"); // assert targetType Assert.assertEquals(MetadataEntity.APPLICATION, MetadataKey.extractTargetType(mdsValueKey.getKey())); Assert.assertEquals(MetadataEntity.APPLICATION, MetadataKey.extractTargetType(mdsIndexKey.getKey())); }
@Test public void testGetTargetTypeChild() { ApplicationId expectedAppId = new ApplicationId("ns1", "app1"); MDSKey mdsValueKey = MetadataKey.createValueRowKey(expectedAppId.toMetadataEntity(), "key1"); ProgramId expectedProgramId = expectedAppId.spark("spark1"); MDSKey mdsValueKey2 = MetadataKey.createValueRowKey(expectedProgramId.toMetadataEntity(), "key2"); // assert that the key for parent child are independent and correct MetadataEntity actualAppId = MetadataKey.extractMetadataEntityFromKey(mdsValueKey.getKey()); Assert.assertEquals(expectedAppId.toMetadataEntity(), actualAppId); MetadataEntity actualProgramId = MetadataKey.extractMetadataEntityFromKey(mdsValueKey2.getKey()); Assert.assertEquals(expectedProgramId.toMetadataEntity(), actualProgramId); }
@Test public void testCrossNamespaceCustomSearch() throws Exception { MetadataDataset dataset = getDataset(DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("crossNamespaceCustom"), MetadataScope.SYSTEM); TransactionExecutor txnl = dsFrameworkUtil.newInMemoryTransactionExecutor((TransactionAware) dataset); String appName = "app"; MetadataEntity ns1App = new NamespaceId("ns1").app(appName).toMetadataEntity(); MetadataEntity ns2App = new NamespaceId("ns2").app(appName).toMetadataEntity(); txnl.execute(() -> { dataset.addProperty(ns2App, AbstractSystemMetadataWriter.ENTITY_NAME_KEY, appName); dataset.addProperty(ns1App, AbstractSystemMetadataWriter.ENTITY_NAME_KEY, appName); }); SortInfo nameAsc = new SortInfo(AbstractSystemMetadataWriter.ENTITY_NAME_KEY, SortInfo.SortOrder.ASC); SearchRequest request = new SearchRequest(null, "*", EnumSet.allOf(EntityTypeSimpleName.class), nameAsc, 0, 10, 0, null, false, EnumSet.allOf(EntityScope.class)); SearchResults results = txnl.execute(() -> dataset.search(request)); List<MetadataEntry> actual = results.getResults(); List<MetadataEntry> expected = new ArrayList<>(); expected.add(new MetadataEntry(ns1App, AbstractSystemMetadataWriter.ENTITY_NAME_KEY, appName)); expected.add(new MetadataEntry(ns2App, AbstractSystemMetadataWriter.ENTITY_NAME_KEY, appName)); Assert.assertEquals(expected, actual); }
@Test public void testCrossNamespaceDefaultSearch() throws Exception { MetadataDataset dataset = getDataset(DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("crossNamespaceDefault"), MetadataScope.SYSTEM); TransactionExecutor txnl = dsFrameworkUtil.newInMemoryTransactionExecutor((TransactionAware) dataset); MetadataEntity ns1App = new NamespaceId("ns1").app("a").toMetadataEntity(); MetadataEntity ns2App = new NamespaceId("ns2").app("a").toMetadataEntity(); txnl.execute(() -> { dataset.addProperty(ns1App, "k1", "v1"); dataset.addProperty(ns1App, "k2", "v2"); dataset.addProperty(ns2App, "k1", "v1"); }); SearchRequest request1 = new SearchRequest(null, "v1", EnumSet.allOf(EntityTypeSimpleName.class), SortInfo.DEFAULT, 0, 10, 0, null, false, EnumSet.allOf(EntityScope.class)); SearchResults results = txnl.execute(() -> dataset.search(request1)); Set<MetadataEntry> actual = new HashSet<>(results.getResults()); Set<MetadataEntry> expected = new HashSet<>(); expected.add(new MetadataEntry(ns1App, "k1", "v1")); expected.add(new MetadataEntry(ns2App, "k1", "v1")); Assert.assertEquals(expected, actual); SearchRequest request2 = new SearchRequest(null, "v2", EnumSet.allOf(EntityTypeSimpleName.class), SortInfo.DEFAULT, 0, 10, 0, null, false, EnumSet.allOf(EntityScope.class)); results = txnl.execute(() -> dataset.search(request2)); Assert.assertEquals(Collections.singletonList(new MetadataEntry(ns1App, "k2", "v2")), results.getResults()); SearchRequest star = new SearchRequest(null, "*", EnumSet.allOf(EntityTypeSimpleName.class), SortInfo.DEFAULT, 0, 10, 0, null, false, EnumSet.allOf(EntityScope.class)); results = txnl.execute(() -> dataset.search(star)); expected.add(new MetadataEntry(ns1App, "k2", "v2")); Assert.assertEquals(expected, new HashSet<>(results.getResults())); }
@Test public void testWorkflowTags() throws Exception { String appName = WorkflowAppWithFork.class.getSimpleName(); ApplicationId appId = NamespaceId.DEFAULT.app(appName); String workflowName = WorkflowAppWithFork.WorkflowWithFork.class.getSimpleName(); ArtifactId artifactId = NamespaceId.DEFAULT.artifact(appId.getApplication(), "1.0"); ApplicationWithPrograms appWithPrograms = createAppWithWorkflow(artifactId, appId, workflowName); WorkflowSpecification workflowSpec = appWithPrograms.getSpecification().getWorkflows().get(workflowName); SystemMetadataWriterStage systemMetadataWriterStage = new SystemMetadataWriterStage(metadataPublisher); StageContext stageContext = new StageContext(Object.class); systemMetadataWriterStage.process(stageContext); systemMetadataWriterStage.process(appWithPrograms); // verify that the workflow is not tagged with the fork node name. First wait for tags to show up Tasks.waitFor(false, () -> metadataStore.getTags( MetadataScope.SYSTEM, appId.workflow(workflowName).toMetadataEntity()).isEmpty(), 5, TimeUnit.SECONDS); Set<String> workflowSystemTags = metadataStore.getTags(MetadataScope.SYSTEM, appId.workflow(workflowName).toMetadataEntity()); Sets.SetView<String> intersection = Sets.intersection(workflowSystemTags, getWorkflowForkNodes(workflowSpec)); Assert.assertTrue("Workflows should not be tagged with fork node names, but found the following fork nodes " + "in the workflow's system tags: " + intersection, intersection.isEmpty()); // verify that metadata was added for the workflow's schedule. First wait for the metadata to show up Tasks.waitFor(false, () -> metadataStore.getProperties( MetadataScope.SYSTEM, appId.toMetadataEntity()).isEmpty(), 5, TimeUnit.SECONDS); Map<String, String> metadataProperties = metadataStore.getMetadata(MetadataScope.SYSTEM, appId.toMetadataEntity()).getProperties(); Assert.assertEquals(WorkflowAppWithFork.SCHED_NAME + ":testDescription", metadataProperties.get("schedule:" + WorkflowAppWithFork.SCHED_NAME)); }
@Test public void testCrossNamespaceCustomSearch() throws Exception { MetadataStorage mds = getMetadataStorage(); String appName = "app"; MetadataEntity ns1App = new NamespaceId("ns1").app(appName).toMetadataEntity(); MetadataEntity ns2App = new NamespaceId("ns2").app(appName).toMetadataEntity(); Metadata meta = new Metadata(SYSTEM, props(ENTITY_NAME_KEY, appName)); MetadataRecord app1Record = new MetadataRecord(ns1App, meta); MetadataRecord app2Record = new MetadataRecord(ns2App, meta); mds.apply(new Update(ns1App, meta)); mds.apply(new Update(ns2App, meta)); assertInOrder(mds, SearchRequest.of("*").setSorting(new Sorting(ENTITY_NAME_KEY, Sorting.Order.ASC)).build(), app1Record, app2Record); // clean up mds.batch(ImmutableList.of(new Drop(ns1App), new Drop(ns2App))); }
NamespaceId ns2 = new NamespaceId("ns2"); MetadataEntity ns1app1 = ns1.app("a1").toMetadataEntity(); MetadataEntity ns1app2 = ns1.app("a2").toMetadataEntity(); MetadataEntity ns1app3 = ns1.app("a3").toMetadataEntity(); MetadataEntity ns2app1 = ns2.app("a1").toMetadataEntity(); MetadataEntity ns2app2 = ns2.app("a2").toMetadataEntity();
@Test public void testCrossNamespaceDefaultSearch() throws IOException { MetadataStorage mds = getMetadataStorage(); NamespaceId ns1 = new NamespaceId("ns1"); NamespaceId ns2 = new NamespaceId("ns2"); MetadataEntity ns1app = ns1.app("a").toMetadataEntity(); MetadataEntity ns2app = ns2.app("a").toMetadataEntity(); MetadataRecord app1Record = new MetadataRecord(ns1app, new Metadata(USER, props("k1", "v1", "k2", "v2"))); MetadataRecord app2Record = new MetadataRecord(ns2app, new Metadata(USER, props("k1", "v1"))); mds.apply(new Update(ns1app, app1Record.getMetadata())); mds.apply(new Update(ns2app, app2Record.getMetadata())); assertResults(mds, SearchRequest.of("v1").build(), app1Record, app2Record); assertResults(mds, SearchRequest.of("v2").build(), app1Record); assertResults(mds, SearchRequest.of("*").build(), app1Record, app2Record); // clean up mds.batch(ImmutableList.of(new Drop(ns1app), new Drop(ns2app))); }
MDSKey mdsValueKey = MetadataKey.createValueRowKey(applicationId1.toMetadataEntity(), "key1"); MetadataEntity actual = MetadataKey.extractMetadataEntityFromKey(mdsValueKey.getKey()); Assert.assertEquals(applicationId1.toMetadataEntity(), actual); mdsValueKey = MetadataKey.createValueRowKey(applicationId2.toMetadataEntity(), "key1"); actual = MetadataKey.extractMetadataEntityFromKey(mdsValueKey.getKey()); Assert.assertEquals(applicationId1.toMetadataEntity(), actual);
NamespaceId ns2Id = new NamespaceId("ns2"); MetadataEntity ns1app1 = ns1Id.app("a1").toMetadataEntity(); MetadataEntity ns1app2 = ns1Id.app("a2").toMetadataEntity(); MetadataEntity ns1app3 = ns1Id.app("a3").toMetadataEntity(); MetadataEntity ns2app1 = ns2Id.app("a1").toMetadataEntity(); MetadataEntity ns2app2 = ns2Id.app("a2").toMetadataEntity();
@Test public void testGetMDSValueKey() { MDSKey mdsValueKey = MetadataKey.createValueRowKey(new ApplicationId("ns1", "app1").toMetadataEntity(), "key1"); MDSKey.Splitter split = mdsValueKey.split(); // skip value key bytes split.skipBytes(); // assert target type is application Assert.assertEquals(MetadataEntity.APPLICATION, split.getString()); // assert key-value pairs Assert.assertEquals(MetadataEntity.NAMESPACE, split.getString()); Assert.assertEquals("ns1", split.getString()); Assert.assertEquals(MetadataEntity.APPLICATION, split.getString()); Assert.assertEquals("app1", split.getString()); Assert.assertEquals("key1", split.getString()); // asert that there is nothing more left in the key Assert.assertFalse(split.hasRemaining()); }
@Test public void testGetMDSIndexKey() { MDSKey mdsIndexKey = MetadataKey.createIndexRowKey(new ApplicationId("ns1", "app1").toMetadataEntity(), "key1", "value1"); MDSKey.Splitter split = mdsIndexKey.split(); // skip value key bytes split.skipBytes(); // assert target type Assert.assertEquals(MetadataEntity.APPLICATION, split.getString()); // assert key-value pairs Assert.assertEquals(MetadataEntity.NAMESPACE, split.getString()); Assert.assertEquals("ns1", split.getString()); Assert.assertEquals(MetadataEntity.APPLICATION, split.getString()); Assert.assertEquals("app1", split.getString()); Assert.assertEquals("key1", split.getString()); Assert.assertEquals("value1", split.getString()); Assert.assertFalse(split.hasRemaining()); } }