@Override public String getTag(NameSpaceContainer value) { switch (value.getType()) { case DATASET: return value.getDataset().getTag(); case FOLDER: return value.getFolder().getTag(); case HOME: return value.getHome().getTag(); case SOURCE: return value.getSource().getTag(); case SPACE: return value.getSpace().getTag(); default: throw new UnsupportedOperationException("Unknown type: " + value.getType()); } }
/** * Helper method that converts the given object into a {@link NamespaceEntity} */ static NamespaceEntity toEntity(Type type, NamespaceKey path, Object config, boolean keyNormalization) { final NameSpaceContainer container = new NameSpaceContainer(); final NamespaceInternalKey namespaceInternalKey = new NamespaceInternalKey(path, keyNormalization); container.setType(type); switch (type) { case DATASET: container.setDataset((DatasetConfig) config); break; case FOLDER: container.setFolder((FolderConfig) config); break; case HOME: container.setHome((HomeConfig) config); break; case SOURCE: container.setSource((SourceConfig) config); break; case SPACE: container.setSpace((SpaceConfig) config); break; default: throw new UnsupportedOperationException("Unknown type: " + type); } container.setFullPathList(path.getPathComponents()); return new NamespaceEntity(namespaceInternalKey, container); } }
protected void doTraverseAndDeleteChildren(final NameSpaceContainer child) throws NamespaceException { final NamespaceInternalKey childKey = new NamespaceInternalKey(new NamespaceKey(child.getFullPathList()), keyNormalization); traverseAndDeleteChildren(childKey, child); switch (child.getType()) { case FOLDER: namespace.delete(childKey.getKey(), child.getFolder().getTag()); break; case DATASET: namespace.delete(childKey.getKey(), child.getDataset().getTag()); break; default: // Only leaf level or intermediate namespace container types are expected here. throw new RuntimeException("Unexpected namespace container type: " + child.getType()); } }
public List<ResourceTreeEntity> listPath(NamespaceKey root, boolean showDatasets) throws NamespaceException, UnsupportedEncodingException { final List<ResourceTreeEntity> resources = Lists.newArrayList(); for (NameSpaceContainer container : namespaceService.get().list(root)) { if (container.getType() == Type.FOLDER) { resources.add(new ResourceTreeEntity(container.getFolder())); } else if (showDatasets && container.getType() == Type.DATASET) { resources.add(new ResourceTreeEntity(container.getDataset())); } } return resources; }
private static DatasetType getType(NamespaceService service, List<String> namespacePath){ try{ List<NameSpaceContainer> containers = service.getEntities(Collections.singletonList(new NamespaceKey(namespacePath))); if(containers != null && !containers.isEmpty()){ NameSpaceContainer container = containers.get(0); if(container.getType() == Type.DATASET){ return container.getDataset().getType(); } } }catch(Exception ex){ } return null; }
@Test public void testGetDatasetCatalogEntityById() throws Exception { DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setId(new EntityId("dataset-id")); datasetConfig.setFullPathList(Collections.singletonList("path")); datasetConfig.setType(VIRTUAL_DATASET); VirtualDataset virtualDataset = new VirtualDataset(); virtualDataset.setSql(""); datasetConfig.setVirtualDataset(virtualDataset); NameSpaceContainer namespaceContainer = new NameSpaceContainer(); namespaceContainer.setType(NameSpaceContainer.Type.DATASET); namespaceContainer.setDataset(datasetConfig); when(namespaceService.getEntityById(datasetConfig.getId().getId())).thenReturn(namespaceContainer); ReflectionSettings reflectionSettings = mock(ReflectionSettings.class); when(reflectionSettings.getStoredReflectionSettings(any(NamespaceKey.class))).thenReturn(Optional.<AccelerationSettings>absent()); when(reflectionServiceHelper.getReflectionSettings()).thenReturn(reflectionSettings); DremioTable dremioTable = mock(DremioTable.class); when(dremioTable.getDatasetConfig()).thenReturn(datasetConfig); when(catalog.getTable(any(String.class))).thenReturn(dremioTable); Optional<CatalogEntity> entity = catalogServiceHelper.getCatalogEntityById(datasetConfig.getId().getId()); assertTrue(entity.isPresent()); CatalogEntity catalogEntity = entity.get(); assertTrue(catalogEntity instanceof Dataset); Dataset dataset = (Dataset) catalogEntity; assertEquals(dataset.getId(), datasetConfig.getId().getId()); }
@GET @Path("parents") @Produces(APPLICATION_JSON) public List<ParentDatasetUI> getParents() throws DatasetNotFoundException, NamespaceException { final VirtualDatasetUI virtualDatasetUI = datasetService.get(datasetPath, version); final List<ParentDatasetUI> parentDatasetUIs = Lists.newArrayList(); final List<NamespaceKey> parentDatasetPaths = Lists.newArrayList(); for (ParentDataset parentDataset : virtualDatasetUI.getParentsList()) { parentDatasetPaths.add(new NamespaceKey(parentDataset.getDatasetPathList())); } for (NameSpaceContainer nameSpaceContainer : datasetService.getNamespaceService().getEntities(parentDatasetPaths)) { if (nameSpaceContainer != null && nameSpaceContainer.getType() == Type.DATASET) { parentDatasetUIs.add(new ParentDatasetUI(nameSpaceContainer.getFullPathList(), nameSpaceContainer.getDataset().getType())); } } return parentDatasetUIs; }
private DatasetConfig saveDataset(List<String> path, DatasetType type, Function<DatasetConfig, DatasetConfig> transformer) throws NamespaceException { final NamespaceKey key = new NamespaceKey(path); final byte[] binaryKey = NamespaceServiceImpl.getKey(key); final Optional<DatasetConfig> oldDataset = Optional.ofNullable(namespaceStore.get(binaryKey)).map(NameSpaceContainer::getDataset); final DatasetConfig datasetConfig = transformer.apply(new DatasetConfig() .setId(oldDataset.map(DatasetConfig::getId).orElse(new EntityId().setId(UUID.randomUUID().toString()))) .setName(path.get(path.size() - 1)) .setFullPathList(path) .setType(type) .setTag(oldDataset.map(DatasetConfig::getTag).orElse(null)) .setOwner("dremio")); final NameSpaceContainer container = new NameSpaceContainer() .setType(NameSpaceContainer.Type.DATASET) .setFullPathList(path) .setDataset(datasetConfig); namespaceStore.put(binaryKey, container); return datasetConfig; }
private void newS3Source(KVStore<byte[], NameSpaceContainer> namespace, String path, S3PluginConfig s3PluginConfig) { final List<String> fullPathList = Arrays.asList(path); final SourceConfig config = new SourceConfig() .setId(new EntityId(UUID.randomUUID().toString())) .setName(path) .setConnectionConf(s3PluginConfig); namespace.put(NamespaceServiceImpl.getKey(new NamespaceKey(fullPathList)), new NameSpaceContainer() .setFullPathList(fullPathList) .setType(NameSpaceContainer.Type.SOURCE) .setSource(config)); } }
@Override public int getAllDatasetsCount(NamespaceKey parent) throws NamespaceException { int count = 0; for (final NameSpaceContainer container : traverseEntity(parent)) { if (container.getType() == DATASET) { ++count; } } return count; }
NameSpaceContainer namespaceContainer = new NameSpaceContainer(); namespaceContainer.setSource(sourceConfig); namespaceContainer.setType(NameSpaceContainer.Type.SOURCE); when(namespaceService.getEntities(Collections.singletonList(new NamespaceKey(sourceConfig.getName())))).thenReturn(Collections.singletonList(namespaceContainer)); folderConfig1.setName("folder1"); folderConfig1.setFullPathList(Arrays.asList(sourceConfig.getName(), folderConfig1.getName())); NameSpaceContainer folder = new NameSpaceContainer(); folder.setFolder(folderConfig1); folder.setType(NameSpaceContainer.Type.FOLDER); folderConfig2.setName("folder2"); folderConfig2.setFullPathList(Arrays.asList(sourceConfig.getName(), folderConfig2.getName())); folder = new NameSpaceContainer(); folder.setFolder(folderConfig2); folder.setType(NameSpaceContainer.Type.FOLDER);
.filter(entry -> NameSpaceContainer.Type.DATASET == entry.getValue().getType()) .forEach(entry -> { DatasetConfig datasetConfig = update(entry.getValue().getDataset()); if (datasetConfig != null) { entry.getValue().setDataset(datasetConfig); namespaceStore.put(entry.getKey(), entry.getValue());
@Override public DatasetConfig apply(Entry<NamespaceKey, NameSpaceContainer> input) { return input.getValue().getDataset(); }}).toSortedList(new Comparator<DatasetConfig>(){ @Override
@Test public void testNamespaceContainerVersionExtractor() throws Exception { NameSpaceContainerVersionExtractor versionExtractor = new NameSpaceContainerVersionExtractor(); NameSpaceContainer container = new NameSpaceContainer(); container.setType(NameSpaceContainer.Type.SOURCE); SourceConfig config = new SourceConfig(); container.setSource(config); // test precommit for sources, which increments the version versionExtractor.preCommit(container); assertEquals(0, config.getConfigOrdinal().longValue()); versionExtractor.preCommit(container); assertEquals(1, config.getConfigOrdinal().longValue()); // test preCommit rollback AutoCloseable autoCloseable = versionExtractor.preCommit(container); assertEquals(2, config.getConfigOrdinal().longValue()); autoCloseable.close(); assertEquals(1, config.getConfigOrdinal().longValue()); } }
private void addHome(String name) throws Exception { final HomeConfig home = new HomeConfig() .setId(new EntityId().setId(UUID.randomUUID().toString())) .setOwner(name); final NameSpaceContainer container = new NameSpaceContainer() .setType(NameSpaceContainer.Type.HOME) .setFullPathList(Arrays.asList(name)) .setHome(home); namespaceStore.put(NamespaceServiceImpl.getKey(new NamespaceKey("@" + name)), container); }
@Test public void testDeleteSpace() throws Exception { Space space = new Space("space-id", "mySpace", "1", 0L, null); NameSpaceContainer namespaceContainer = new NameSpaceContainer(); namespaceContainer.setSpace(catalogServiceHelper.getSpaceConfig(space)); namespaceContainer.setType(NameSpaceContainer.Type.SPACE); when(namespaceService.getEntityById(space.getId())).thenReturn(namespaceContainer); catalogServiceHelper.deleteCatalogItem(space.getId(), space.getTag()); verify(namespaceService, times(1)).deleteSpace(new NamespaceKey(space.getName()), space.getTag()); }
for(Map.Entry<byte[], NameSpaceContainer> entry : namespace.find()) { NameSpaceContainer container = entry.getValue(); switch(container.getType()) { case SOURCE: { final SourceConfig source = container.getSource(); sourceConfigs.put(source.getName(), source); continue; final DatasetConfig dataset = container.getDataset(); final ReadDefinition readDefinition = dataset.getReadDefinition(); if (readDefinition == null || readDefinition.getSplitVersion() == null) {
private Collection<NameSpaceContainer> traverseEntity(final NamespaceKey root) throws NamespaceException { final LinkedList<NameSpaceContainer> toBeTraversed = new LinkedList<>(listEntity(root)); final LinkedList<NameSpaceContainer> visited = new LinkedList<>(); while (!toBeTraversed.isEmpty()) { final NameSpaceContainer container = toBeTraversed.removeFirst(); if (NamespaceUtils.isListable(container.getType())) { toBeTraversed.addAll(listEntity(new NamespaceKey(container.getFullPathList()))); } visited.add(container); } return visited; }
@Override public AutoCloseable preCommit(NameSpaceContainer value) { if (value.getType() != NameSpaceContainer.Type.SOURCE) { return AutoCloseables.noop(); } // for sources, we want to maintain a numeric version so we can distinguish chronological order SourceConfig source = value.getSource(); Long configOrdinal= source.getConfigOrdinal(); value.getSource().setConfigOrdinal(configOrdinal == null ? 0 : configOrdinal + 1); return () -> { // rollback the ordinal in case the commit fails value.getSource().setConfigOrdinal(configOrdinal); }; }
private final String getDescription(NameSpaceContainer container) { String description = null; switch (container.getType()) { case SOURCE: description = container.getSource().getDescription(); break; case SPACE: description = container.getSpace().getDescription(); break; case HOME: // for home space we should pre populate wiki with default text description = "# Wikis & Tags\n" + "\n" + "![Gnarly Catalog](https://d33wubrfki0l68.cloudfront.net/c1a54376c45a9276c080f3d10ed25ce61c17bcd2/2b946/img/home/open-source-for-everyone.svg)\n" + "\n" + "Starting with Dremio 3.0, you can now:\n" + "* Add wikis to datasets, spaces and sources.\n" + "* Tag datasets and search using tags.\n" + "\n" + "\n" + "You are reading the wiki for your home space! This sidebar always shows the wiki for the current source, space or folder you are browsing.\n" + "\n" + "When previewing datasets, click on the `Catalog` tab to create a wiki or add tags to that dataset.\n" + "\n" + "**Tip:** You can hide the wiki by clicking on the sidebar icon on upper right hand side."; break; default: break; } return description; }