private void removeNullOrEmptyTags(final DatasetId metadataDatasetInstance, final MetadataScope scope) { execute(dataset -> { dataset.removeNullOrEmptyTags(metadataDatasetInstance.toMetadataEntity()); }, scope, DATASET.V2); }
@Override public boolean isUpgradeRequired(MetadataScope scope) { DatasetId datasetId = getMetadataDatasetInstance(scope); Set<String> tags = getTags(scope, datasetId.toMetadataEntity()); // Check if you are in the process of an Upgrade if (tags.contains(NEEDS_UPGRADE_TAG)) { LOG.debug("NEEDS_UPGRADE_TAG found on Metadata Dataset. Upgrade is required."); return true; } // If no tag was found or Version tag does not match current version boolean versionTagFound = false; for (String tag: tags) { if (tag.startsWith(VERSION_TAG_PREFIX)) { versionTagFound = true; String datasetVersion = getVersionFromVersionTag(tag); if (!datasetVersion.equals(ProjectInfo.getVersion().toString())) { LOG.debug("Metadata Dataset version mismatch. Needs Upgrade"); removeTags(scope, datasetId.toMetadataEntity(), Collections.singleton(tag)); addTags(scope, datasetId.toMetadataEntity(), NEEDS_UPGRADE_TAG_SET); return true; } } } if (!versionTagFound) { addTags(scope, datasetId.toMetadataEntity(), NEEDS_UPGRADE_TAG_SET); return true; } return false; }
@Override public void markUpgradeComplete(MetadataScope scope) { DatasetId datasetId = getMetadataDatasetInstance(scope); LOG.info("Add Upgrade tag with version {} to {}", ProjectInfo.getVersion().toString(), datasetId); addTags(scope, datasetId.toMetadataEntity(), getTagWithVersion(ProjectInfo.getVersion().toString())); removeTags(scope, datasetId.toMetadataEntity(), NEEDS_UPGRADE_TAG_SET); }
/** * Drops a dataset. * * @param spec specification of dataset to be dropped. * @throws Exception on error. */ private void dropDataset(DatasetId instance, DatasetSpecification spec) throws Exception { LOG.info("Deleting dataset {}.{}", instance.getNamespace(), instance.getEntityName()); disableExplore(instance, spec); if (!instanceManager.delete(instance)) { throw new DatasetNotFoundException(instance); } metaCache.invalidate(instance); // by pass the auth check for dataset type when the operation is not creation DatasetTypeMeta typeMeta = getTypeInfo(instance.getParent(), spec.getType(), true); if (typeMeta == null) { throw new DatasetNotFoundException(instance); } opExecutorClient.drop(instance, typeMeta, spec); // Remove metadata for the dataset LOG.trace("Removing metadata for dataset {}", instance); metadataStore.removeMetadata(instance.toMetadataEntity()); LOG.trace("Removed metadata for dataset {}", instance); publishAudit(instance, AuditType.DELETE); // deletes the owner principal for the entity if one was stored during creation ownerAdmin.delete(instance); }
/** * Drops a dataset. * * @param spec specification of dataset to be dropped. * @throws Exception on error. */ private void dropDataset(DatasetId instance, DatasetSpecification spec) throws Exception { LOG.info("Deleting dataset {}.{}", instance.getNamespace(), instance.getEntityName()); disableExplore(instance, spec); if (!instanceManager.delete(instance)) { throw new DatasetNotFoundException(instance); } metaCache.invalidate(instance); // by pass the auth check for dataset type when the operation is not creation DatasetTypeMeta typeMeta = getTypeInfo(instance.getParent(), spec.getType(), true); if (typeMeta == null) { throw new DatasetNotFoundException(instance); } opExecutorClient.drop(instance, typeMeta, spec); // Remove metadata for the dataset LOG.trace("Removing metadata for dataset {}", instance); metadataPublisher.publish(NamespaceId.SYSTEM, new MetadataOperation.Drop(instance.toMetadataEntity())); LOG.trace("Removed metadata for dataset {}", instance); publishAudit(instance, AuditType.DELETE); // deletes the owner principal for the entity if one was stored during creation ownerAdmin.delete(instance); }
private void generateMetadataUpdates() { store.addTags(MetadataScope.USER, dataset.toMetadataEntity(), datasetTags); store.addProperties(MetadataScope.USER, app.toMetadataEntity(), appProperties); store.addTags(MetadataScope.USER, app.toMetadataEntity(), appTags); store.addTags(MetadataScope.USER, service.toMetadataEntity(), tags); store.removeTags(MetadataScope.USER, service.toMetadataEntity()); store.removeTags(MetadataScope.USER, dataset.toMetadataEntity(), datasetTags); store.removeMetadata(MetadataScope.USER, app.toMetadataEntity()); }
store.addTags(MetadataScope.USER, service1.toMetadataEntity(), userTags); store.addTags(MetadataScope.SYSTEM, service1.toMetadataEntity(), sysTags); store.addTags(MetadataScope.USER, dataset2.toMetadataEntity(), streamUserTags); store.removeTags(MetadataScope.USER, dataset2.toMetadataEntity(), streamUserTags); store.addProperties(MetadataScope.USER, dataset2.toMetadataEntity(), userProps); store.removeProperties(MetadataScope.USER, dataset2.toMetadataEntity(), ImmutableSet.of("key1", "key2", "multiword")); store.addProperties(MetadataScope.USER, dataset2.toMetadataEntity(), streamUserProps); store.addProperties(MetadataScope.USER, dataset1.toMetadataEntity(), datasetUserProps);
NamespaceId nsId = new NamespaceId(ns); MetadataEntity service1 = nsId.app("app1").service("service1").toMetadataEntity(); MetadataEntity dataset1 = nsId.dataset("ds1").toMetadataEntity(); MetadataEntity dataset2 = nsId.dataset("ds2").toMetadataEntity();
@Test public void testMesssages() { // test dataset InvalidMetadataException invalidMetadataException = new InvalidMetadataException(NamespaceId.DEFAULT.dataset("ds").toMetadataEntity(), "error"); String expectedMessage = "Unable to set metadata for dataset: ds " + "which exists in namespace: default. error"; Assert.assertEquals(expectedMessage, invalidMetadataException.getMessage()); // test program invalidMetadataException = new InvalidMetadataException(NamespaceId.DEFAULT.app("app").program(ProgramType.WORKER, "wk") .toMetadataEntity(), "error"); expectedMessage = "Unable to set metadata for worker: wk in application: app of version: -SNAPSHOT deployed in " + "namespace: default. error"; Assert.assertEquals(expectedMessage, invalidMetadataException.getMessage()); // test custom entity MetadataEntity customEntity = MetadataEntity.builder(NamespaceId.DEFAULT.dataset("ds").toMetadataEntity()) .appendAsType("field", "empName").build(); invalidMetadataException = new InvalidMetadataException(customEntity, "error"); expectedMessage = "Unable to set metadata for namespace=default,dataset=ds,field=empName of type 'field'. error"; Assert.assertEquals(expectedMessage, invalidMetadataException.getMessage()); } }
MetadataEntity service = nsId.app("app").service("service").toMetadataEntity(); MetadataEntity worker = nsId.app("app2").worker("worker").toMetadataEntity(); MetadataEntity dataset = nsId.dataset("dataset").toMetadataEntity(); MetadataEntity trackerDataset = nsId.dataset("_auditLog").toMetadataEntity();
TableProperties.setRowFieldName(props, "name"); DatasetId datasetId = NamespaceId.DEFAULT.dataset("file"); MetadataEntity entity = datasetId.toMetadataEntity(); datasetFramework.addInstance("table", datasetId, props.build());
metadataStore.addProperties(MetadataScope.USER, dataset1.toMetadataEntity(), run1Data1Meta.getProperties()); metadataStore.addTags(MetadataScope.USER, dataset1.toMetadataEntity(), run1Data1Meta.getTags()); metadataStore.addProperties(MetadataScope.USER, dataset2.toMetadataEntity(), run1Data2Meta.getProperties()); metadataStore.addTags(MetadataScope.USER, dataset2.toMetadataEntity(), run1Data2Meta.getTags());
@Test public void testSearchOnTypes() throws Exception { MetadataStorage mds = getMetadataStorage(); MetadataEntity myDs = NamespaceId.DEFAULT.dataset("myDs").toMetadataEntity(); MetadataEntity myField1 = MetadataEntity.builder(myDs).appendAsType("field", "myField1").build(); MetadataEntity myField2 = MetadataEntity.builder(myDs).appendAsType("field", "myField2").build(); MetadataRecord record1 = new MetadataRecord(myField1, new Metadata(USER, props("testKey1", "testValue1"))); MetadataRecord record2 = new MetadataRecord(myField2, new Metadata(USER, props("testKey2", "testValue2"))); mds.apply(new Update(myField1, record1.getMetadata())); mds.apply(new Update(myField2, record2.getMetadata())); // Search for it based on value assertResults(mds, SearchRequest.of("field:myField1").build(), record1); // should return both fields assertResults(mds, SearchRequest.of("field:myFie*").build(), record1, record2); assertResults(mds, SearchRequest.of("field*").build(), record1, record2); // clean up mds.batch(ImmutableList.of(new Drop(myField1), new Drop(myField2))); }
final MetadataEntity app2 = app2Id.toMetadataEntity(); final MetadataEntity program1 = app1Id.worker("wk1").toMetadataEntity(); final MetadataEntity dataset1 = ns1Id.dataset("ds1").toMetadataEntity(); final MetadataEntity dataset2 = ns1Id.dataset("ds2").toMetadataEntity(); final MetadataEntity file1 = MetadataEntity.builder(dataset1).appendAsType("file", "f1").build();
final ApplicationId appId = nsId.app("app1"); final MetadataEntity program = appId.worker("wk1").toMetadataEntity(); final MetadataEntity dataset = nsId.dataset("ds2").toMetadataEntity();
MetadataStorage mds = getMetadataStorage(); MetadataEntity entity = NamespaceId.DEFAULT.dataset("entity").toMetadataEntity();
final ApplicationId appId = nsId.app("app1"); final MetadataEntity program = appId.worker("wk1").toMetadataEntity(); final MetadataEntity dataset = nsId.dataset("ds2").toMetadataEntity();
MetadataEntity service = nsId.app("app").service("service").toMetadataEntity(); MetadataEntity worker = nsId.app("app2").worker("worker").toMetadataEntity(); MetadataEntity dataset = nsId.dataset("dataset").toMetadataEntity(); MetadataEntity hidden = nsId.dataset("_auditLog").toMetadataEntity();
MetadataStorage mds = getMetadataStorage(); MetadataEntity entity = NamespaceId.DEFAULT.dataset("entity").toMetadataEntity();
final MetadataEntity app = app1Id.toMetadataEntity(); final MetadataEntity program = app1Id.worker(programName).toMetadataEntity(); final MetadataEntity dataset = ns1Id.dataset("ds").toMetadataEntity();