private boolean isDatasetExplorable(DatasetId datasetInstance) { return !NamespaceId.SYSTEM.getNamespace().equals(datasetInstance.getNamespace()) && !"system.queue.config".equals(datasetInstance.getDataset()) && !datasetInstance.getDataset().startsWith("system.sharded.queue") && !datasetInstance.getDataset().startsWith("system.queue") && !datasetInstance.getDataset().startsWith("system.stream"); }
static DatasetId toDatasetInstanceId(String namespace, String name) throws BadRequestException { try { return new DatasetId(namespace, name); } catch (IllegalArgumentException | NullPointerException e) { throw new BadRequestException(e.getMessage()); } }
protected QueryHandle doDisableExploreDataset(DatasetId datasetInstance, DatasetSpecification spec) throws ExploreException { String body = spec == null ? null : GSON.toJson(new DisableExploreParameters(spec)); String endpoint = spec == null ? "disable" : "disable-internal"; HttpResponse response = doPost(String.format("namespaces/%s/data/explore/datasets/%s/%s", datasetInstance.getNamespace(), datasetInstance.getEntityName(), endpoint), body, null); if (response.getResponseCode() == HttpURLConnection.HTTP_OK) { return QueryHandle.fromId(parseResponseAsMap(response, "handle")); } throw new ExploreException(String.format("Cannot disable explore on dataset %s. Reason: %s", datasetInstance.toString(), response)); }
@SuppressWarnings("unused") public static DatasetId fromIdParts(Iterable<String> idString) { Iterator<String> iterator = idString.iterator(); return new DatasetId(next(iterator, "namespace"), nextAndEnd(iterator, "dataset")); }
public static boolean isSystemDatasetInUserNamespace(DatasetId datasetInstanceId) { return !NamespaceId.SYSTEM.equals(datasetInstanceId.getParent()) && ("system.queue.config".equals(datasetInstanceId.getEntityName()) || datasetInstanceId.getEntityName().startsWith("system.sharded.queue") || datasetInstanceId.getEntityName().startsWith("system.queue") || datasetInstanceId.getEntityName().startsWith("system.stream")); }
private void addDataset(MDSKey.Builder keyBuilder, DatasetId datasetInstance) { keyBuilder.add(DATASET_MARKER) .add(datasetInstance.getNamespace()) .add(datasetInstance.getEntityName()); }
@VisibleForTesting void clear() throws IOException { service.dropTable(PREVIEW_TABLE_ID.getDataset()); service.ensureTableExists(PREVIEW_TABLE_ID.getDataset()); } }
/** * Drops a dataset. * * @param spec specification of dataset to be dropped. * @throws Exception on error. */ private void dropDataset(DatasetId instance, DatasetSpecification spec) throws Exception { LOG.info("Deleting dataset {}.{}", instance.getNamespace(), instance.getEntityName()); disableExplore(instance, spec); if (!instanceManager.delete(instance)) { throw new DatasetNotFoundException(instance); } metaCache.invalidate(instance); // by pass the auth check for dataset type when the operation is not creation DatasetTypeMeta typeMeta = getTypeInfo(instance.getParent(), spec.getType(), true); if (typeMeta == null) { throw new DatasetNotFoundException(instance); } opExecutorClient.drop(instance, typeMeta, spec); // Remove metadata for the dataset LOG.trace("Removing metadata for dataset {}", instance); metadataStore.removeMetadata(instance.toMetadataEntity()); LOG.trace("Removed metadata for dataset {}", instance); publishAudit(instance, AuditType.DELETE); // deletes the owner principal for the entity if one was stored during creation ownerAdmin.delete(instance); }
@Override @Nullable public DatasetId fromNamespaced(DatasetId datasetInstanceId) { String namespacedDatasetName = datasetInstanceId.getEntityName(); if (!contains(namespacedDatasetName, datasetInstanceId.getNamespace())) { return null; } String prefix = rootPrefix + datasetInstanceId.getNamespace() + "."; String nonNamespaced = namespacedDatasetName.substring(prefix.length()); return datasetInstanceId.getParent().dataset(nonNamespaced); }
@Test public void testWildCardEntities() throws Exception { // in-mem authorizer does not support wildcard privileges so we can't actually enforce after grant. // this test is just to check that our grant apis work fine. After grant we list privileges to check for existence DatasetId datasetId = new DatasetId("ns", "wildcard"); String wildcardEntityStar = datasetId.toString().replace("wildcard", "*"); client.grant(Authorizable.fromString(wildcardEntityStar), admin, ImmutableSet.of(Action.ADMIN)); String wildcardEntityQuestion = datasetId.toString().replace("wildcard", "someSt?"); client.grant(Authorizable.fromString(wildcardEntityQuestion), admin, ImmutableSet.of(Action.ADMIN)); Set<Privilege> privileges = client.listPrivileges(admin); Assert.assertTrue(privileges.contains(new Privilege(Authorizable.fromString(wildcardEntityStar), Action.ADMIN))); Assert.assertTrue(privileges.contains(new Privilege(Authorizable.fromString(wildcardEntityQuestion), Action.ADMIN))); // revoke privilege from the wildcard entity and it should not exist anymore client.revoke(Authorizable.fromString(wildcardEntityQuestion), admin, ImmutableSet.of(Action.ADMIN)); privileges = client.listPrivileges(admin); Assert.assertFalse(privileges.contains(new Privilege(Authorizable.fromString(wildcardEntityQuestion), Action.ADMIN))); // revoke all should work too String authFromPrivilege = null; for (Privilege privilege : privileges) { if (privilege.getAuthorizable().getEntityType().equals(EntityType.DATASET)) { authFromPrivilege = privilege.getAuthorizable().toString(); } } client.revoke(Authorizable.fromString(authFromPrivilege)); privileges = client.listPrivileges(admin); Assert.assertFalse(privileges.contains(new Privilege(Authorizable.fromString(wildcardEntityStar), Action.ADMIN))); }
public static Notification forPartitions(DatasetId datasetId, Collection<? extends PartitionKey> partitionKeys) { Map<String, String> properties = new HashMap<>(); properties.put(DATASET_ID, datasetId.toString()); properties.put(NUM_PARTITIONS, Integer.toString(partitionKeys.size())); properties.put(PARTITION_KEYS, GSON.toJson(partitionKeys)); return new Notification(Notification.Type.PARTITION, properties); }
private void removeNullOrEmptyTags(final DatasetId metadataDatasetInstance, final MetadataScope scope) { execute(dataset -> { dataset.removeNullOrEmptyTags(metadataDatasetInstance.toMetadataEntity()); }, scope, DATASET.V2); }
@Override public void drop(DatasetId datasetInstanceId, DatasetTypeMeta typeMeta, DatasetSpecification spec) throws Exception { DatasetType type = client.getDatasetType(typeMeta, null, new ConstantClassLoaderProvider()); if (type == null) { throw new IllegalArgumentException("Dataset type cannot be instantiated for provided type meta: " + typeMeta); } DatasetAdmin admin = type.getAdmin(DatasetContext.from(datasetInstanceId.getNamespace()), spec); admin.drop(); }
@Override public boolean hasInstance(DatasetId datasetInstanceId) { readLock.lock(); try { return instances.contains(datasetInstanceId.getParent(), datasetInstanceId); } finally { readLock.unlock(); } }
@Override public void perform(Arguments arguments, PrintStream output) throws Exception { DatasetId dataset = cliConfig.getCurrentNamespace().dataset(arguments.get(ArgumentName.DATASET.toString())); datasetClient.delete(dataset); output.printf("Successfully deleted dataset instance '%s'\n", dataset.getEntityName()); }
private DatasetId getMappedDatasetInstance(DatasetId datasetInstanceId) { if (datasetNameMapping.containsKey(datasetInstanceId.getEntityName())) { return datasetInstanceId.getParent().dataset(datasetNameMapping.get(datasetInstanceId.getEntityName())); } return datasetInstanceId; } }
private void addDataset(MDSKey.Builder keyBuilder, DatasetId datasetInstance) { keyBuilder.add(DATASET_MARKER) .add(datasetInstance.getNamespace()) .add(datasetInstance.getEntityName()); }
@Inject DefaultPreviewStore(LevelDBTableService service) { try { this.service = service; service.ensureTableExists(PREVIEW_TABLE_ID.getDataset()); this.table = new LevelDBTableCore(PREVIEW_TABLE_ID.getDataset(), service); } catch (IOException e) { throw new RuntimeException("Error creating preview table", e); } }
/** * Drops a dataset. * * @param spec specification of dataset to be dropped. * @throws Exception on error. */ private void dropDataset(DatasetId instance, DatasetSpecification spec) throws Exception { LOG.info("Deleting dataset {}.{}", instance.getNamespace(), instance.getEntityName()); disableExplore(instance, spec); if (!instanceManager.delete(instance)) { throw new DatasetNotFoundException(instance); } metaCache.invalidate(instance); // by pass the auth check for dataset type when the operation is not creation DatasetTypeMeta typeMeta = getTypeInfo(instance.getParent(), spec.getType(), true); if (typeMeta == null) { throw new DatasetNotFoundException(instance); } opExecutorClient.drop(instance, typeMeta, spec); // Remove metadata for the dataset LOG.trace("Removing metadata for dataset {}", instance); metadataPublisher.publish(NamespaceId.SYSTEM, new MetadataOperation.Drop(instance.toMetadataEntity())); LOG.trace("Removed metadata for dataset {}", instance); publishAudit(instance, AuditType.DELETE); // deletes the owner principal for the entity if one was stored during creation ownerAdmin.delete(instance); }
@Override @Nullable public DatasetId fromNamespaced(DatasetId datasetInstanceId) { String namespacedDatasetName = datasetInstanceId.getEntityName(); if (!contains(namespacedDatasetName, datasetInstanceId.getNamespace())) { return null; } String prefix = rootPrefix + datasetInstanceId.getNamespace() + "."; String nonNamespaced = namespacedDatasetName.substring(prefix.length()); return datasetInstanceId.getParent().dataset(nonNamespaced); }