@Override public void upgrade(UpgradeContext context) throws Exception { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); try { for (SourceConfig source : namespaceService.getSources()) { if (!"HIVE".equalsIgnoreCase(ConnectionReader.toType(source))) { continue; } System.out.printf(" Handling Hive source %s%n", source.getName()); for (NamespaceKey datasetPath : namespaceService.getAllDatasets(new NamespaceKey(source.getName()))) { final DatasetConfig datasetConfig = namespaceService.getDataset(datasetPath); if (datasetConfig.getReadDefinition() == null || datasetConfig.getReadDefinition().getExtendedProperty() == null) { continue; } System.out.printf(" Clearing read definition of table %s%n", datasetPath.getSchemaPath()); datasetConfig.setReadDefinition(null); namespaceService.addOrUpdateDataset(datasetPath, datasetConfig); } } } catch (NamespaceException e) { throw new RuntimeException("Hive121BasedInputSplits failed", e); } }
private void assertNoDatasetsAfterSourceDeletion() throws Exception { final SourceConfig sourceConfig = namespaceService.getSource(mockUpKey); namespaceService.deleteSource(mockUpKey, sourceConfig.getTag()); assertEquals(0, Iterables.size(namespaceService.getAllDatasets(mockUpKey))); }
@Override public void upgrade(UpgradeContext context) { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); try { final NamespaceKey key = new DatasetPath(ImmutableList.of("sys", "materializations")).toNamespaceKey(); final DatasetConfig dataset = namespaceService.getDataset(key); namespaceService.deleteDataset(key, dataset.getTag()); } catch (NamespaceNotFoundException e) { // no metadata was found for sys.materializations // most likely the table was never queried // nothing more to do System.out.println(" 'sys.materializations' metadata not found...skipping"); } catch (NamespaceException e) { throw new RuntimeException("Failed to delete metadata for 'sys.materialization'", e); } } }
protected CatalogEntity createSpace(Space space, NamespaceAttribute... attributes) throws NamespaceException { String spaceName = space.getName(); Preconditions.checkArgument(space.getId() == null, "Space id is immutable."); Preconditions.checkArgument(spaceName != null, "Space name is required."); Preconditions.checkArgument(spaceName.trim().length() > 0, "Space name cannot be empty."); // TODO: move the space name validation somewhere reusable instead of having to create a new SpaceName new SpaceName(spaceName); NamespaceKey namespaceKey = new NamespaceKey(spaceName); // check if space already exists with the given name. if (namespaceService.exists(namespaceKey, NameSpaceContainer.Type.SPACE)) { throw new ConcurrentModificationException(String.format("A space with the name [%s] already exists.", spaceName)); } namespaceService.addOrUpdateSpace(namespaceKey, getSpaceConfig(space), attributes); return getSpaceFromConfig(namespaceService.getSpace(namespaceKey), null); }
/** * returns a Qlik load script for the dataset * @return * @throws DatasetNotFoundException * @throws NamespaceException */ @GET @Produces(TEXT_PLAIN_QLIK_APP) public DatasetConfig get() throws DatasetNotFoundException, NamespaceException { return namespaceService.getDataset(datasetPath.toNamespaceKey()); } }
final List<NamespaceKey> allDatasets = Lists.newArrayList(namespace.getAllDatasets(key)); DatasetConfig datasetConfig = namespace.getDataset(datasetKey); PhysicalDataset physicalDataset = datasetConfig.getPhysicalDataset(); namespace.addOrUpdateDataset(datasetKey, datasetConfig);
@Override public SourceConfig getSource(String username, String sourcename) throws NamespaceException { return factoryProvider.get() .get(username) .getSource(new NamespaceKey(sourcename)); } }
private CheckPoint checkPoint() throws Exception { CheckPoint checkPoint = new CheckPoint(); NamespaceService namespaceService = newNamespaceService(); UserService userService = l(UserService.class); JobsService jobsService = l(JobsService.class); checkPoint.sources = namespaceService.getSources(); checkPoint.spaces = namespaceService.getSpaces(); checkPoint.homes = namespaceService.getHomeSpaces(); checkPoint.users = Arrays.asList(Iterables.toArray(userService.getAllUsers(10000), User.class)); checkPoint.datasets = Lists.newArrayList(); checkPoint.virtualDatasetVersions = Lists.newArrayList(); checkPoint.accelerations = Lists.newArrayList(); /** DX-4498 for (NamespaceKey ds : namespaceService.getAllDatasets(new NamespaceKey(""))) { DatasetConfig datasetConfig = namespaceService.getDataset(ds); checkPoint.datasets.add(datasetConfig); checkPoint.virtualDatasetVersions.addAll( Arrays.asList(Iterables.toArray(datasetService.getAllVersions(new DatasetPath(ds.getPathComponents())), VirtualDatasetUI.class))); if (datasetConfig.getAccelerationId() != null) { checkPoint.accelerations.add(accelerationService.getAccelerationById(new AccelerationId(datasetConfig.getAccelerationId()))); } } */ checkPoint.jobs = ImmutableList.copyOf(jobsService.getAllJobs(null, null, null, 0, Integer.MAX_VALUE, "tshiran")); return checkPoint; }
protected static DatasetConfig addJson(DatasetPath path) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return nsService.getDataset(path.toNamespaceKey()); }
private DatasetConfig getConfig(final String datasetId) { return userNamespaceService.findDatasetByUUID(datasetId); }
protected CatalogEntity createDataset(Dataset dataset, NamespaceAttribute... attributes) throws NamespaceException { validateDataset(dataset); // only handle VDS Preconditions.checkArgument(dataset.getType() != Dataset.DatasetType.PHYSICAL_DATASET, "Physical Datasets can only be created by promoting other entities."); Preconditions.checkArgument(dataset.getId() == null, "Dataset id is immutable."); // verify we can save NamespaceKey topLevelKey = new NamespaceKey(dataset.getPath().get(0)); NamespaceKey parentNamespaceKey = new NamespaceKey(dataset.getPath().subList(0, dataset.getPath().size() - 1)); NamespaceKey namespaceKey = new NamespaceKey(dataset.getPath()); Preconditions.checkArgument(namespaceService.exists(parentNamespaceKey), String.format("Dataset parent path [%s] doesn't exist.", parentNamespaceKey.toString())); // can only create VDs in a space or home NameSpaceContainer entity = getNamespaceEntity(topLevelKey); List<NameSpaceContainer.Type> types = Arrays.asList(NameSpaceContainer.Type.SPACE, NameSpaceContainer.Type.HOME); Preconditions.checkArgument(types.contains(entity.getType()), "Virtual datasets can only be saved into spaces or home space."); sabotContext.getViewCreator(context.getUserPrincipal().getName()).createView(dataset.getPath(), dataset.getSql(), dataset.getSqlContext(), attributes); DatasetConfig created = namespaceService.getDataset(namespaceKey); return getDatasetFromConfig(created, null); }
protected Folder createFolder(Folder folder, NamespaceAttribute... attributes) throws NamespaceException { NamespaceKey parentKey = new NamespaceKey(folder.getPath().subList(0, folder.getPath().size() - 1)); List<NameSpaceContainer> entities = namespaceService.getEntities(Collections.singletonList(parentKey)); NameSpaceContainer container = entities.get(0); if (container == null) { // if we can't find it by id, maybe its not in the namespace throw new IllegalArgumentException(String.format("Could not find entity with path [%s].", folder.getPath())); } NamespaceKey key = new NamespaceKey(folder.getPath()); switch (container.getType()) { case SPACE: case HOME: case FOLDER: { namespaceService.addOrUpdateFolder(key, getFolderConfig(folder), attributes); break; } default: { throw new UnsupportedOperationException(String.format("Can not create a folder inside a [%s].", container.getType())); } } return getFolderFromConfig(namespaceService.getFolder(key), null); }
private boolean refreshSourceNames() throws NamespaceException { final Set<NamespaceKey> foundKeys = Sets.newHashSet(systemUserNamespaceService.getAllDatasets(sourceKey)); boolean changed = false; try { for (SourceTableDefinition accessor : plugin.get() .getDatasets(SYSTEM_USERNAME, msp.getDefaultRetrievalOptions())) { // names only added, never removed. Removal can be done by the full refresh (refreshSource()) if (!foundKeys.contains(accessor.getName()) && accessor.isSaveable()) { try { saver.shallowSave(accessor); changed = true; } catch (ConcurrentModificationException ex) { logger.debug("Concurrent update, ignoring.", ex); } } } // persist last refresh for across restarts. SourceInternalData srcData = sourceDataStore.get(sourceKey); if(srcData == null) { srcData = new SourceInternalData(); srcData.setName(sourceKey.getRoot()); } srcData.setLastNameRefreshDateMs(System.currentTimeMillis()); sourceDataStore.put(sourceKey, srcData); } catch (Exception ex){ logger.warn("Failure while attempting to update metadata for source {}. Terminating update of this source.", sourceKey, ex); } return changed; }
@Override public void upgrade(UpgradeContext context) throws Exception { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); List<SourceConfig> sources = namespaceService.getSources(); for (SourceConfig sourceConfig : sources) { // Pre-1.5, the config object for internal sources was null. Deleting the internal sources without a config will // ensure that they get recreated on startup. if (sourceConfig.getConfig() == null) { System.out.printf(" deleting '%s'%n", sourceConfig.getName()); // following may throw an exception, we let it propagate to fail the upgrade namespaceService.deleteSource(sourceConfig.getKey(), sourceConfig.getTag()); } } }
@Override public List<SourceConfig> getSources(String username) { return factoryProvider.get() .get(username) .getSources(); }
@Override public void upgrade(UpgradeContext context) throws Exception { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); List<SourceConfig> sources = namespaceService.getSources(); for (SourceConfig sourceConfig : sources) { ConnectionConf<?, ?> connectionConf = context.getConnectionReader().getConnectionConf(sourceConfig); if (connectionConf instanceof S3PluginConfig) { S3PluginConfig s3PluginConfig = (S3PluginConfig) connectionConf; if ((s3PluginConfig.credentialType == AWSAuthenticationType.ACCESS_KEY) && isNullOrEmpty(s3PluginConfig.accessKey)) { s3PluginConfig.credentialType = AWSAuthenticationType.NONE; sourceConfig.setConnectionConf(s3PluginConfig); namespaceService.addOrUpdateSource(sourceConfig.getKey(), sourceConfig); } } } }
private boolean storeSchema(NamespaceKey key, DatasetConfig config) throws NamespaceException { try { namespace.addOrUpdateDataset(key, config); return true; } catch (final ConcurrentModificationException ex) { return false; } } }
public void renameDataset(DatasetPath ds, DatasetPath dsNew) { NamespaceKey key = ds.toNamespaceKey(); DatasetConfig config; try { config = getNamespaceService().getDataset(key); if (config != null) { getNamespaceService().renameDataset(key, dsNew.toNamespaceKey()); } }catch (NamespaceException e) { } }
@Test public void testByPath() throws Exception { Source createdSource = createSource(); // test getting a source by name Source source = expectSuccess(getBuilder(getPublicAPI(3).path(CATALOG_PATH).path("by-path").path(createdSource.getName())).buildGet(), new GenericType<Source>() {}); assertEquals(source.getId(), createdSource.getId()); // test getting a folder by path expectSuccess(getBuilder(getPublicAPI(3).path(CATALOG_PATH).path("by-path").path(createdSource.getName()).path("json")).buildGet(), new GenericType<Folder>() {}); // test getting a file with a url character in name (?) expectSuccess(getBuilder(getPublicAPI(3).path(CATALOG_PATH).path("by-path").path(createdSource.getName()).path("testfiles").path("file_with_?.json")).buildGet(), new GenericType<File>() {}); newNamespaceService().deleteSource(new NamespaceKey(source.getName()), source.getTag()); }
public void deleteSource() { NamespaceKey key = new SourcePath(TEST_SOURCE).toNamespaceKey(); SourceConfig config; try { config = getNamespaceService().getSource(key); if (config != null) { getNamespaceService().deleteSource(key, config.getTag()); } } catch (NamespaceException e) { } }