@Override public void deleteModule(DatasetModuleId moduleId) { // TODO (CDAP-6297): check if existing datasets or modules use this module writeLock.lock(); try { moduleClasses.remove(moduleId.getParent(), moduleId); LinkedHashSet<String> availableModuleClasses = getAvailableModuleClasses(moduleId.getParent()); // this will cleanup types DatasetDefinitionRegistry registry = createRegistry(availableModuleClasses, registries.getClass().getClassLoader()); registries.put(moduleId.getParent(), registry); } finally { writeLock.unlock(); } }
@Override public void deleteModule(DatasetModuleId moduleId) { // TODO (CDAP-6297): check if existing datasets or modules use this module writeLock.lock(); try { moduleClasses.remove(moduleId.getParent(), moduleId); LinkedHashSet<String> availableModuleClasses = getAvailableModuleClasses(moduleId.getParent()); // this will cleanup types DatasetDefinitionRegistry registry = createRegistry(availableModuleClasses, registries.getClass().getClassLoader()); registries.put(moduleId.getParent(), registry); } finally { writeLock.unlock(); } }
@Override public void deleteModule(DatasetModuleId moduleId) throws DatasetManagementException { clientCache.getUnchecked(moduleId.getParent()).deleteModule(moduleId.getEntityName()); }
@Override public void deleteModule(DatasetModuleId moduleId) throws DatasetManagementException { clientCache.getUnchecked(moduleId.getParent()).deleteModule(moduleId.getEntityName()); }
@Override public void addModule(DatasetModuleId moduleId, DatasetModule module, Location jarLocation) throws DatasetManagementException { clientCache.getUnchecked(moduleId.getParent()) .addModule(moduleId.getEntityName(), DatasetModules.getDatasetModuleClass(module).getName(), jarLocation); }
@Override public void addModule(DatasetModuleId moduleId, DatasetModule module, Location jarLocation) throws DatasetManagementException { clientCache.getUnchecked(moduleId.getParent()) .addModule(moduleId.getEntityName(), DatasetModules.getDatasetModuleClass(module).getName(), jarLocation); }
@Override public void addModule(DatasetModuleId moduleId, DatasetModule module) throws ModuleConflictException { // TODO (CDAP-6297): check if existing modules overlap, or if this removes a type other modules depend on writeLock.lock(); try { DatasetDefinitionRegistry registry = registries.get(moduleId.getParent()); if (registry == null) { registry = registryFactory.create(); registries.put(moduleId.getParent(), registry); } TypesTrackingRegistry trackingRegistry = new TypesTrackingRegistry(registry); module.register(trackingRegistry); String moduleClassName = DatasetModules.getDatasetModuleClass(module).getName(); moduleClasses.put(moduleId.getParent(), moduleId, moduleClassName); List<String> types = trackingRegistry.getTypes(); nonDefaultTypes.putAll(moduleId.getParent(), types); for (String type : types) { this.types.put(moduleId.getParent().datasetType(type), new DatasetTypeMeta(type, Collections.singletonList( new DatasetModuleMeta(moduleId.getEntityName(), moduleClassName, null, types, Collections.<String>emptyList())))); } } finally { writeLock.unlock(); } }
@Override public void addModule(DatasetModuleId moduleId, DatasetModule module) throws ModuleConflictException { // TODO (CDAP-6297): check if existing modules overlap, or if this removes a type other modules depend on writeLock.lock(); try { DatasetDefinitionRegistry registry = registries.get(moduleId.getParent()); if (registry == null) { registry = registryFactory.create(); registries.put(moduleId.getParent(), registry); } TypesTrackingRegistry trackingRegistry = new TypesTrackingRegistry(registry); module.register(trackingRegistry); String moduleClassName = DatasetModules.getDatasetModuleClass(module).getName(); moduleClasses.put(moduleId.getParent(), moduleId, moduleClassName); List<String> types = trackingRegistry.getTypes(); nonDefaultTypes.putAll(moduleId.getParent(), types); for (String type : types) { this.types.put(moduleId.getParent().datasetType(type), new DatasetTypeMeta(type, Collections.singletonList( new DatasetModuleMeta(moduleId.getEntityName(), moduleClassName, null, types, Collections.<String>emptyList())))); } } finally { writeLock.unlock(); } }
/** * Returns the {@link DatasetModuleMeta metadata} of the specified {@link DatasetModuleId}. */ @Override public DatasetModuleMeta getModule(DatasetModuleId datasetModuleId) throws Exception { ensureNamespaceExists(datasetModuleId.getParent()); DatasetModuleMeta moduleMeta = typeManager.getModule(datasetModuleId); if (moduleMeta == null) { throw new DatasetModuleNotFoundException(datasetModuleId); } return moduleMeta; }
/** * Returns the {@link DatasetModuleMeta metadata} of the specified {@link DatasetModuleId}. */ @Override public DatasetModuleMeta getModule(DatasetModuleId datasetModuleId) throws Exception { ensureNamespaceExists(datasetModuleId.getParent()); DatasetModuleMeta moduleMeta = typeManager.getModule(datasetModuleId); if (moduleMeta == null) { throw new DatasetModuleNotFoundException(datasetModuleId); } return moduleMeta; }
/** * Adds a new {@link DatasetModule}. * * @param datasetModuleId the {@link DatasetModuleId} for the module to be added * @param className the module class name specified in the HTTP header * @param forceUpdate if true, an update will be allowed even if there are conflicts with other modules, or if * removal of a type would break other modules' dependencies * @return a {@link BodyConsumer} to upload the module jar in chunks * @throws NotFoundException if the namespace in which the module is being added is not found * @throws IOException if there are issues while performing I/O like creating temporary directories, moving/unpacking * module jar files * @throws DatasetModuleConflictException if #forceUpdate is {@code false}, and there are conflicts with other modules */ @Override public BodyConsumer addModule(final DatasetModuleId datasetModuleId, final String className, final boolean forceUpdate) throws Exception { NamespaceId namespaceId = datasetModuleId.getParent(); ensureNamespaceExists(namespaceId); // It is now determined that a new dataset module will be deployed. First grant privileges, then deploy the module. // If creation fails, revoke the granted privileges. This ensures that just like delete, there may be orphaned // privileges in rare scenarios, but there can never be orphaned datasets. // If the module previously existed and was deleted, but revoking privileges somehow failed, there may be orphaned // privileges for the module. Revoke them first, so no users unintentionally get privileges on the dataset. return createModuleConsumer(datasetModuleId, className, forceUpdate); }
/** * Adds a new {@link DatasetModule}. * * @param datasetModuleId the {@link DatasetModuleId} for the module to be added * @param className the module class name specified in the HTTP header * @param forceUpdate if true, an update will be allowed even if there are conflicts with other modules, or if * removal of a type would break other modules' dependencies * @return a {@link BodyConsumer} to upload the module jar in chunks * @throws NotFoundException if the namespace in which the module is being added is not found * @throws IOException if there are issues while performing I/O like creating temporary directories, moving/unpacking * module jar files * @throws DatasetModuleConflictException if #forceUpdate is {@code false}, and there are conflicts with other modules */ @Override public BodyConsumer addModule(final DatasetModuleId datasetModuleId, final String className, final boolean forceUpdate) throws Exception { NamespaceId namespaceId = datasetModuleId.getParent(); ensureNamespaceExists(namespaceId); // It is now determined that a new dataset module will be deployed. First grant privileges, then deploy the module. // If creation fails, revoke the granted privileges. This ensures that just like delete, there may be orphaned // privileges in rare scenarios, but there can never be orphaned datasets. // If the module previously existed and was deleted, but revoking privileges somehow failed, there may be orphaned // privileges for the module. Revoke them first, so no users unintentionally get privileges on the dataset. return createModuleConsumer(datasetModuleId, className, forceUpdate); }
@Override public void addModule(DatasetModuleId moduleId, DatasetModule module) throws DatasetManagementException { Class<?> moduleClass = DatasetModules.getDatasetModuleClass(module); try { Location deploymentJar = createDeploymentJar(moduleClass); try { clientCache.getUnchecked(moduleId.getParent()) .addModule(moduleId.getEntityName(), moduleClass.getName(), deploymentJar); } finally { try { deploymentJar.delete(); } catch (IOException e) { // Just log warning, since the add module operation can still proceed LOG.warn("Failed to delete temporary deployment jar {}", deploymentJar, e); } } } catch (IOException e) { String msg = String.format("Could not create jar for deploying dataset module %s with main class %s", moduleId, moduleClass.getName()); LOG.error(msg, e); throw new DatasetManagementException(msg, e); } }
@Override public void addModule(DatasetModuleId moduleId, DatasetModule module) throws DatasetManagementException { Class<?> moduleClass = DatasetModules.getDatasetModuleClass(module); try { Location deploymentJar = createDeploymentJar(moduleClass); try { clientCache.getUnchecked(moduleId.getParent()) .addModule(moduleId.getEntityName(), moduleClass.getName(), deploymentJar); } finally { try { deploymentJar.delete(); } catch (IOException e) { // Just log warning, since the add module operation can still proceed LOG.warn("Failed to delete temporary deployment jar {}", deploymentJar, e); } } } catch (IOException e) { String msg = String.format("Could not create jar for deploying dataset module %s with main class %s", moduleId, moduleClass.getName()); LOG.error(msg, e); throw new DatasetManagementException(msg, e); } }
/** * Checks if a dataset module exists. * * @param module the dataset module to check * @throws IOException if a network error occurred * @throws UnauthenticatedException if the request is not authorized successfully in the gateway server */ public boolean exists(DatasetModuleId module) throws IOException, UnauthenticatedException, UnauthorizedException { URL url = config.resolveNamespacedURLV3(module.getParent(), String.format("data/modules/%s", module.getModule())); HttpResponse response = restClient.execute(HttpMethod.GET, url, config.getAccessToken(), HttpURLConnection.HTTP_NOT_FOUND); return response.getResponseCode() != HttpURLConnection.HTTP_NOT_FOUND; }
/** * Deletes the specified {@link DatasetModuleId} */ @Override public void delete(DatasetModuleId datasetModuleId) throws Exception { NamespaceId namespaceId = datasetModuleId.getParent(); if (NamespaceId.SYSTEM.equals(namespaceId)) { throw new UnsupportedOperationException( String.format("Cannot delete module '%s' from '%s' namespace.", datasetModuleId.getModule(), datasetModuleId.getNamespace())); } ensureNamespaceExists(namespaceId); DatasetModuleMeta moduleMeta = typeManager.getModule(datasetModuleId); if (moduleMeta == null) { throw new DatasetModuleNotFoundException(datasetModuleId); } try { typeManager.deleteModule(datasetModuleId); } catch (DatasetModuleConflictException e) { throw new DatasetModuleCannotBeDeletedException(datasetModuleId, e.getMessage()); } }
/** * Gets information about a dataset module. * * @param module the dataset module * @return {@link DatasetModuleMeta} of the dataset module * @throws DatasetModuleNotFoundException if the dataset module with the specified name was not found * @throws IOException if a network error occurred * @throws UnauthenticatedException if the request is not authorized successfully in the gateway server */ public DatasetModuleMeta get(DatasetModuleId module) throws DatasetModuleNotFoundException, IOException, UnauthenticatedException, UnauthorizedException { URL url = config.resolveNamespacedURLV3(module.getParent(), String.format("data/modules/%s", module.getModule())); HttpResponse response = restClient.execute(HttpMethod.GET, url, config.getAccessToken(), HttpURLConnection.HTTP_NOT_FOUND); if (response.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { throw new DatasetModuleNotFoundException(module); } return ObjectResponse.fromJsonBody(response, DatasetModuleMeta.class).getResponseObject(); } }
/** * Deletes the specified {@link DatasetModuleId} */ @Override public void delete(DatasetModuleId datasetModuleId) throws Exception { NamespaceId namespaceId = datasetModuleId.getParent(); if (NamespaceId.SYSTEM.equals(namespaceId)) { throw new UnsupportedOperationException( String.format("Cannot delete module '%s' from '%s' namespace.", datasetModuleId.getModule(), datasetModuleId.getNamespace())); } ensureNamespaceExists(namespaceId); DatasetModuleMeta moduleMeta = typeManager.getModule(datasetModuleId); if (moduleMeta == null) { throw new DatasetModuleNotFoundException(datasetModuleId); } try { typeManager.deleteModule(datasetModuleId); } catch (DatasetModuleConflictException e) { throw new DatasetModuleCannotBeDeletedException(datasetModuleId, e.getMessage()); } }
/** * Adds a new dataset module. * * @param module the new dataset module * @param className name of the dataset module class within the moduleJarFile * @param moduleJarFile Jar file containing the dataset module class and dependencies * @throws BadRequestException if the moduleJarFile does not exist * @throws AlreadyExistsException if a dataset module with the same name already exists * @throws IOException if a network error occurred */ public void add(DatasetModuleId module, String className, File moduleJarFile) throws BadRequestException, AlreadyExistsException, IOException, UnauthenticatedException { URL url = config.resolveNamespacedURLV3(module.getParent(), String.format("data/modules/%s", module.getModule())); Map<String, String> headers = ImmutableMap.of("X-Class-Name", className); HttpRequest request = HttpRequest.put(url).addHeaders(headers).withBody(moduleJarFile).build(); HttpResponse response = restClient.upload(request, config.getAccessToken(), HttpURLConnection.HTTP_BAD_REQUEST, HttpURLConnection.HTTP_CONFLICT); if (response.getResponseCode() == HttpURLConnection.HTTP_BAD_REQUEST) { throw new BadRequestException(String.format("Module jar file does not exist: %s", moduleJarFile)); } else if (response.getResponseCode() == HttpURLConnection.HTTP_CONFLICT) { throw new DatasetModuleAlreadyExistsException(module); } }
/** * Deletes a dataset module. * * @param module the dataset module to delete * @throws DatasetModuleCannotBeDeletedException if the dataset module cannot be deleted, * usually due to other dataset modules or dataset instances using the dataset module * @throws DatasetModuleNotFoundException if the dataset module with the specified name was not found * @throws IOException if a network error occurred * @throws UnauthenticatedException if the request is not authorized successfully in the gateway server */ public void delete(DatasetModuleId module) throws DatasetModuleCannotBeDeletedException, DatasetModuleNotFoundException, IOException, UnauthenticatedException, UnauthorizedException { URL url = config.resolveNamespacedURLV3(module.getParent(), String.format("data/modules/%s", module.getModule())); HttpResponse response = restClient.execute(HttpMethod.DELETE, url, config.getAccessToken(), HttpURLConnection.HTTP_CONFLICT, HttpURLConnection.HTTP_NOT_FOUND); if (response.getResponseCode() == HttpURLConnection.HTTP_CONFLICT) { throw new DatasetModuleCannotBeDeletedException(module); } else if (response.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { throw new DatasetModuleNotFoundException(module); } }