/** * Checks if the specified type exists in the 'system' namespace * * @return true if type exists in the 'system' namespace, false otherwise * @throws DatasetManagementException * @throws ServiceUnavailableException when the dataset service is not running */ default boolean hasSystemType(String typeName) throws DatasetManagementException { return hasType(NamespaceId.SYSTEM.datasetType(typeName)); }
@Override public EntityId apply(DatasetTypeMeta input) { return namespaceId.datasetType(input.getName()); } }, null);
@Override public EntityId apply(DatasetTypeMeta input) { return namespaceId.datasetType(input.getName()); } }, null);
/** * Checks if the specified type exists in the 'system' namespace * * @return true if type exists in the 'system' namespace, false otherwise * @throws DatasetManagementException * @throws ServiceUnavailableException when the dataset service is not running */ default boolean hasSystemType(String typeName) throws DatasetManagementException { return hasType(NamespaceId.SYSTEM.datasetType(typeName)); }
public void datasetTypeClient() throws Exception { // Construct the client used to interact with CDAP DatasetTypeClient datasetTypeClient = new DatasetTypeClient(clientConfig); // Fetch the dataset type information using the type name DatasetTypeMeta datasetTypeMeta = datasetTypeClient.get(NamespaceId.DEFAULT.datasetType("someDatasetType")); // Fetch the dataset type information using the classname datasetTypeClient.get(NamespaceId.DEFAULT.datasetType(SomeDataset.class.getName())); }
@GET @Path("/data/types/{name}") public void getTypeInfo(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("name") String name) throws Exception { responder.sendJson(HttpResponseStatus.OK, GSON.toJson(typeService.getType(new NamespaceId(namespaceId).datasetType(name)))); } }
@GET @Path("/data/types/{name}") public void getTypeInfo(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("name") String name) throws Exception { responder.sendJson(HttpResponseStatus.OK, GSON.toJson(typeService.getType(new NamespaceId(namespaceId).datasetType(name)))); } }
@Override public boolean hasType(String datasetTypeName) { return registry.hasType(datasetTypeName) || datasetTypeMDS.getType(getNamespaceId().datasetType(datasetTypeName)) != null; } }
@Override public boolean hasType(String datasetTypeName) { return registry.hasType(datasetTypeName) || datasetTypeMDS.getType(getNamespaceId().datasetType(datasetTypeName)) != null; } }
private ObjectResponse<DatasetTypeMeta> getType(String typeName) throws IOException { return getType(NamespaceId.DEFAULT.datasetType(typeName)); }
private HttpResponse getMissingType(String typeName) throws IOException { return makeTypeInfoRequest(NamespaceId.DEFAULT.datasetType(typeName)); }
@Override public void add(DatasetDefinition def) { String typeName = def.getName(); DatasetTypeId typeId = getNamespaceId().datasetType(typeName); DatasetTypeMeta existingType = datasetTypeMDS.getType(typeId); if (existingType != null) { DatasetModuleMeta existingModule = existingType.getModules().get(existingType.getModules().size() - 1); // we allow redefining an existing type if // - it was previously defined by the same module (i.e., this is an upgrade of that module) // - it is a forced update and the existing type is not a system type if (!moduleBeingAdded.getEntityName().equals(existingModule.getName()) && (!tolerateConflicts || NamespaceId.SYSTEM.getNamespace().equals(existingModule.getName()))) { throw new TypeConflictException(String.format( "Attempt to add dataset module '%s' containing dataset type '%s' that already exists in module '%s'", moduleBeingAdded.getEntityName(), typeName, existingModule.getName())); } } types.add(typeName); registry.add(def); }
@Override public void add(DatasetDefinition def) { String typeName = def.getName(); DatasetTypeId typeId = getNamespaceId().datasetType(typeName); DatasetTypeMeta existingType = datasetTypeMDS.getType(typeId); if (existingType != null) { DatasetModuleMeta existingModule = existingType.getModules().get(existingType.getModules().size() - 1); // we allow redefining an existing type if // - it was previously defined by the same module (i.e., this is an upgrade of that module) // - it is a forced update and the existing type is not a system type if (!moduleBeingAdded.getEntityName().equals(existingModule.getName()) && (!tolerateConflicts || NamespaceId.SYSTEM.getNamespace().equals(existingModule.getName()))) { throw new TypeConflictException(String.format( "Attempt to add dataset module '%s' containing dataset type '%s' that already exists in module '%s'", moduleBeingAdded.getEntityName(), typeName, existingModule.getName())); } } types.add(typeName); registry.add(def); }
@Test public void checkDatasetType() throws DatasetManagementException { DatasetFramework dsFramework = getInjector().getInstance(DatasetFramework.class); Assert.assertTrue(dsFramework.hasType(NamespaceId.SYSTEM.datasetType(Schedulers.STORE_TYPE_NAME))); }
@Test public void testCrossNSSpark() throws Exception { createAuthNamespace(); ApplicationId appId = AUTH_NAMESPACE.app(TestSparkCrossNSDatasetApp.APP_NAME); Map<EntityId, Set<Action>> neededPrivileges = ImmutableMap.<EntityId, Set<Action>>builder() .put(appId, EnumSet.of(Action.ADMIN)) .put(AUTH_NAMESPACE.artifact(TestSparkCrossNSDatasetApp.class.getSimpleName(), "1.0-SNAPSHOT"), EnumSet.of(Action.ADMIN)) .put(AUTH_NAMESPACE.dataset(TestSparkCrossNSDatasetApp.DEFAULT_OUTPUT_DATASET), EnumSet.of(Action.ADMIN)) .put(AUTH_NAMESPACE.datasetType(KeyValueTable.class.getName()), EnumSet.of(Action.ADMIN)) .build(); setUpPrivilegeAndRegisterForDeletion(ALICE, neededPrivileges); ProgramId programId = appId.spark(TestSparkCrossNSDatasetApp.SPARK_PROGRAM_NAME); // bob will be executing the program grantAndAssertSuccess(programId, BOB, EnumSet.of(Action.EXECUTE)); cleanUpEntities.add(programId); ApplicationManager appManager = deployApplication(AUTH_NAMESPACE, TestSparkCrossNSDatasetApp.class); SparkManager sparkManager = appManager.getSparkManager(TestSparkCrossNSDatasetApp.SparkCrossNSDatasetProgram .class.getSimpleName()); testCrossNSSystemDatasetAccessWithAuthSpark(sparkManager); testCrossNSDatasetAccessWithAuthSpark(sparkManager); }
@Override public void perform(Arguments arguments, PrintStream output) throws Exception { DatasetTypeId type = cliConfig.getCurrentNamespace().datasetType( arguments.get(ArgumentName.DATASET_TYPE.toString())); DatasetTypeMeta datasetTypeMeta = datasetTypeClient.get(type); Table table = Table.builder() .setHeader("name", "modules") .setRows(ImmutableList.of(datasetTypeMeta), new RowMaker<DatasetTypeMeta>() { @Override public List<?> makeRow(DatasetTypeMeta object) { return Lists.newArrayList(object.getName(), Joiner.on(", ").join(object.getModules())); } }).build(); cliConfig.getTableRenderer().render(cliConfig, output, table); }
@Override public void perform(Arguments arguments, PrintStream output) throws Exception { DatasetTypeId type = cliConfig.getCurrentNamespace().datasetType( arguments.get(ArgumentName.DATASET_TYPE.toString())); DatasetTypeMeta datasetTypeMeta = datasetTypeClient.get(type); Table table = Table.builder() .setHeader("name", "modules") .setRows(ImmutableList.of(datasetTypeMeta), new RowMaker<DatasetTypeMeta>() { @Override public List<?> makeRow(DatasetTypeMeta object) { return Lists.newArrayList(object.getName(), Joiner.on(", ").join(object.getModules())); } }).build(); cliConfig.getTableRenderer().render(cliConfig, output, table); }
public void writeModule(NamespaceId namespaceId, DatasetModuleMeta moduleMeta) { DatasetModuleId datasetModuleId = namespaceId.datasetModule(moduleMeta.getName()); DatasetModuleMeta existing = getModule(datasetModuleId); write(getModuleKey(namespaceId.getEntityName(), moduleMeta.getName()), moduleMeta); for (String type : moduleMeta.getTypes()) { writeTypeToModuleMapping(namespaceId.datasetType(type), datasetModuleId); } if (existing != null) { Set<String> removed = new HashSet<>(existing.getTypes()); removed.removeAll(moduleMeta.getTypes()); for (String type : removed) { deleteAll(getTypeKey(datasetModuleId.getNamespace(), type)); } } }
public void writeModule(NamespaceId namespaceId, DatasetModuleMeta moduleMeta) { DatasetModuleId datasetModuleId = namespaceId.datasetModule(moduleMeta.getName()); DatasetModuleMeta existing = getModule(datasetModuleId); write(getModuleKey(namespaceId.getEntityName(), moduleMeta.getName()), moduleMeta); for (String type : moduleMeta.getTypes()) { writeTypeToModuleMapping(namespaceId.datasetType(type), datasetModuleId); } if (existing != null) { Set<String> removed = new HashSet<>(existing.getTypes()); removed.removeAll(moduleMeta.getTypes()); for (String type : removed) { deleteAll(getTypeKey(datasetModuleId.getNamespace(), type)); } } }
@Test public void testNotFound() throws Exception { NamespaceId nonExistent = new NamespaceId("nonExistent"); HttpResponse response = makeModulesRequest(nonExistent); assertNamespaceNotFound(response, nonExistent); response = makeTypesRequest(nonExistent); assertNamespaceNotFound(response, nonExistent); DatasetModuleId datasetModule = nonExistent.datasetModule("module"); response = makeModuleInfoRequest(datasetModule); assertNamespaceNotFound(response, nonExistent); DatasetTypeId datasetType = nonExistent.datasetType("type"); response = makeTypeInfoRequest(datasetType); assertNamespaceNotFound(response, nonExistent); response = deployModule(datasetModule, TestModule1.class); assertNamespaceNotFound(response, nonExistent); response = deleteModule(datasetModule); assertNamespaceNotFound(response, nonExistent); response = deleteModules(nonExistent); assertNamespaceNotFound(response, nonExistent); }