public NameSpaceContainer newMessage() { return new NameSpaceContainer(); }
@Test public void testCreateVDSInSourceShouldFail() throws Exception { when(namespaceService.exists(any(NamespaceKey.class))).thenReturn(true); // provide a source container NameSpaceContainer namespaceContainer = new NameSpaceContainer(); namespaceContainer.setType(NameSpaceContainer.Type.SOURCE); when(namespaceService.getEntities(Matchers.anyList())).thenReturn(Collections.singletonList(namespaceContainer)); // VDS needs sql Dataset dataset = new Dataset( null, Dataset.DatasetType.VIRTUAL_DATASET, Arrays.asList("source", "path"), null, null, null, null, "sql", null, null, null ); thrown.expect(IllegalArgumentException.class); catalogServiceHelper.createCatalogItem(dataset); }
private DatasetConfig saveDataset(List<String> path, DatasetType type, Function<DatasetConfig, DatasetConfig> transformer) throws NamespaceException { final NamespaceKey key = new NamespaceKey(path); final byte[] binaryKey = NamespaceServiceImpl.getKey(key); final Optional<DatasetConfig> oldDataset = Optional.ofNullable(namespaceStore.get(binaryKey)).map(NameSpaceContainer::getDataset); final DatasetConfig datasetConfig = transformer.apply(new DatasetConfig() .setId(oldDataset.map(DatasetConfig::getId).orElse(new EntityId().setId(UUID.randomUUID().toString()))) .setName(path.get(path.size() - 1)) .setFullPathList(path) .setType(type) .setTag(oldDataset.map(DatasetConfig::getTag).orElse(null)) .setOwner("dremio")); final NameSpaceContainer container = new NameSpaceContainer() .setType(NameSpaceContainer.Type.DATASET) .setFullPathList(path) .setDataset(datasetConfig); namespaceStore.put(binaryKey, container); return datasetConfig; }
@Test public void testDeleteFolder() throws Exception { Folder folder = new Folder("folder-id", Arrays.asList("source", "folder"), "1", null); NameSpaceContainer namespaceContainer = new NameSpaceContainer(); namespaceContainer.setFolder(CatalogServiceHelper.getFolderConfig(folder)); namespaceContainer.setType(NameSpaceContainer.Type.FOLDER); when(namespaceService.getEntityById(folder.getId())).thenReturn(namespaceContainer); catalogServiceHelper.deleteCatalogItem(folder.getId(), folder.getTag()); verify(namespaceService, times(1)).deleteFolder(new NamespaceKey(folder.getPath()), folder.getTag()); }
@Test public void testDeleteSpace() throws Exception { Space space = new Space("space-id", "mySpace", "1", 0L, null); NameSpaceContainer namespaceContainer = new NameSpaceContainer(); namespaceContainer.setSpace(catalogServiceHelper.getSpaceConfig(space)); namespaceContainer.setType(NameSpaceContainer.Type.SPACE); when(namespaceService.getEntityById(space.getId())).thenReturn(namespaceContainer); catalogServiceHelper.deleteCatalogItem(space.getId(), space.getTag()); verify(namespaceService, times(1)).deleteSpace(new NamespaceKey(space.getName()), space.getTag()); }
/** * Helper method that converts the given object into a {@link NamespaceEntity} */ static NamespaceEntity toEntity(Type type, NamespaceKey path, Object config, boolean keyNormalization) { final NameSpaceContainer container = new NameSpaceContainer(); final NamespaceInternalKey namespaceInternalKey = new NamespaceInternalKey(path, keyNormalization); container.setType(type); switch (type) { case DATASET: container.setDataset((DatasetConfig) config); break; case FOLDER: container.setFolder((FolderConfig) config); break; case HOME: container.setHome((HomeConfig) config); break; case SOURCE: container.setSource((SourceConfig) config); break; case SPACE: container.setSpace((SpaceConfig) config); break; default: throw new UnsupportedOperationException("Unknown type: " + type); } container.setFullPathList(path.getPathComponents()); return new NamespaceEntity(namespaceInternalKey, container); } }
private void addHome(String name) throws Exception { final HomeConfig home = new HomeConfig() .setId(new EntityId().setId(UUID.randomUUID().toString())) .setOwner(name); final NameSpaceContainer container = new NameSpaceContainer() .setType(NameSpaceContainer.Type.HOME) .setFullPathList(Arrays.asList(name)) .setHome(home); namespaceStore.put(NamespaceServiceImpl.getKey(new NamespaceKey("@" + name)), container); }
private void newS3Source(KVStore<byte[], NameSpaceContainer> namespace, String path, S3PluginConfig s3PluginConfig) { final List<String> fullPathList = Arrays.asList(path); final SourceConfig config = new SourceConfig() .setId(new EntityId(UUID.randomUUID().toString())) .setName(path) .setConnectionConf(s3PluginConfig); namespace.put(NamespaceServiceImpl.getKey(new NamespaceKey(fullPathList)), new NameSpaceContainer() .setFullPathList(fullPathList) .setType(NameSpaceContainer.Type.SOURCE) .setSource(config)); } }
@Test public void testGetDatasetCatalogEntityById() throws Exception { DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setId(new EntityId("dataset-id")); datasetConfig.setFullPathList(Collections.singletonList("path")); datasetConfig.setType(VIRTUAL_DATASET); VirtualDataset virtualDataset = new VirtualDataset(); virtualDataset.setSql(""); datasetConfig.setVirtualDataset(virtualDataset); NameSpaceContainer namespaceContainer = new NameSpaceContainer(); namespaceContainer.setType(NameSpaceContainer.Type.DATASET); namespaceContainer.setDataset(datasetConfig); when(namespaceService.getEntityById(datasetConfig.getId().getId())).thenReturn(namespaceContainer); ReflectionSettings reflectionSettings = mock(ReflectionSettings.class); when(reflectionSettings.getStoredReflectionSettings(any(NamespaceKey.class))).thenReturn(Optional.<AccelerationSettings>absent()); when(reflectionServiceHelper.getReflectionSettings()).thenReturn(reflectionSettings); DremioTable dremioTable = mock(DremioTable.class); when(dremioTable.getDatasetConfig()).thenReturn(datasetConfig); when(catalog.getTable(any(String.class))).thenReturn(dremioTable); Optional<CatalogEntity> entity = catalogServiceHelper.getCatalogEntityById(datasetConfig.getId().getId()); assertTrue(entity.isPresent()); CatalogEntity catalogEntity = entity.get(); assertTrue(catalogEntity instanceof Dataset); Dataset dataset = (Dataset) catalogEntity; assertEquals(dataset.getId(), datasetConfig.getId().getId()); }
@Test public void testGetSpaceCatalogEntityById() throws Exception { SpaceConfig spaceConfig = new SpaceConfig(); spaceConfig.setId(new EntityId("space-id")); spaceConfig.setName("mySpace"); NameSpaceContainer namespaceContainer = new NameSpaceContainer(); namespaceContainer.setType(NameSpaceContainer.Type.SPACE); namespaceContainer.setSpace(spaceConfig); when(namespaceService.getEntityById(spaceConfig.getId().getId())).thenReturn(namespaceContainer); when(namespaceService.getEntities(Collections.singletonList(new NamespaceKey(spaceConfig.getName())))).thenReturn(Collections.singletonList(namespaceContainer)); // for children listing, we just send the space back to keep it simple when(namespaceService.list(new NamespaceKey(spaceConfig.getName()))).thenReturn(Collections.singletonList(namespaceContainer)); Optional<CatalogEntity> catalogEntityById = catalogServiceHelper.getCatalogEntityById(spaceConfig.getId().getId()); assertTrue(catalogEntityById.isPresent()); CatalogEntity catalogEntity = catalogEntityById.get(); assertTrue(catalogEntity instanceof Space); Space space = (Space) catalogEntity; assertEquals(space.getId(), spaceConfig.getId().getId()); assertEquals(space.getChildren().size(), 1); }
when(namespaceService.findDatasetByUUID(dataset.getId())).thenReturn(config); NameSpaceContainer namespaceContainer = new NameSpaceContainer(); namespaceContainer.setType(NameSpaceContainer.Type.DATASET); namespaceContainer.setDataset(config);
@Test public void testNamespaceContainerVersionExtractor() throws Exception { NameSpaceContainerVersionExtractor versionExtractor = new NameSpaceContainerVersionExtractor(); NameSpaceContainer container = new NameSpaceContainer(); container.setType(NameSpaceContainer.Type.SOURCE); SourceConfig config = new SourceConfig(); container.setSource(config); // test precommit for sources, which increments the version versionExtractor.preCommit(container); assertEquals(0, config.getConfigOrdinal().longValue()); versionExtractor.preCommit(container); assertEquals(1, config.getConfigOrdinal().longValue()); // test preCommit rollback AutoCloseable autoCloseable = versionExtractor.preCommit(container); assertEquals(2, config.getConfigOrdinal().longValue()); autoCloseable.close(); assertEquals(1, config.getConfigOrdinal().longValue()); } }
@Test public void testDeleteDataset() throws Exception { Dataset dataset = new Dataset( "dataset-id", Dataset.DatasetType.VIRTUAL_DATASET, Arrays.asList("source", "path"), null, 0L, "1", null, "sql", null, null, null ); NameSpaceContainer namespaceContainer = new NameSpaceContainer(); namespaceContainer.setType(NameSpaceContainer.Type.DATASET); DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setId(new EntityId(dataset.getId())); datasetConfig.setType(VIRTUAL_DATASET); datasetConfig.setFullPathList(dataset.getPath()); datasetConfig.setTag(dataset.getTag()); namespaceContainer.setDataset(datasetConfig); when(namespaceService.getEntityById(dataset.getId())).thenReturn(namespaceContainer); DremioTable dremioTable = mock(DremioTable.class); when(dremioTable.getDatasetConfig()).thenReturn(datasetConfig); when(catalog.getTable(any(String.class))).thenReturn(dremioTable); catalogServiceHelper.deleteCatalogItem(dataset.getId(), "1"); verify(namespaceService, times(1)).deleteDataset(new NamespaceKey(dataset.getPath()), datasetConfig.getTag()); }
when(namespaceService.findDatasetByUUID(dataset.getId())).thenReturn(config); NameSpaceContainer namespaceContainer = new NameSpaceContainer(); namespaceContainer.setType(NameSpaceContainer.Type.DATASET); namespaceContainer.setDataset(config);
@Test public void testDeleteSource() throws Exception { SourceConfig sourceConfig = new SourceConfig(); sourceConfig.setName("mySource"); sourceConfig.setId(new EntityId("source-id")); sourceConfig.setTag("1"); sourceConfig.setType("NAS"); AccelerationSettings settings = new AccelerationSettings(); settings.setGracePeriod(1L); settings.setRefreshPeriod(1L); Source source = new Source(sourceConfig, settings, mock(ConnectionReader.class)); NameSpaceContainer namespaceContainer = new NameSpaceContainer(); namespaceContainer.setType(NameSpaceContainer.Type.SOURCE); namespaceContainer.setSource(sourceConfig); when(namespaceService.getEntityById(source.getId())).thenReturn(namespaceContainer); catalogServiceHelper.deleteCatalogItem(source.getId(), "1"); verify(sourceService, times(1)).deleteSource(sourceConfig); }
private void addSource(String name) throws Exception { final SourceConfig source = new SourceConfig() .setId(new EntityId().setId(UUID.randomUUID().toString())) .setName(name) .setType("test") .setCtime(100L) .setAccelerationRefreshPeriod(REFRESH_PERIOD_MS) .setAccelerationGracePeriod(GRACE_PERIOD_MS) .setMetadataPolicy(new MetadataPolicy().setDatasetDefinitionExpireAfterMs(TimeUnit.DAYS.toMillis(1))); final NameSpaceContainer container = new NameSpaceContainer() .setType(NameSpaceContainer.Type.SOURCE) .setFullPathList(Arrays.asList(name)) .setSource(source); namespaceStore.put(NamespaceServiceImpl.getKey(new NamespaceKey(name)), container); }
NameSpaceContainer namespaceContainer = new NameSpaceContainer(); namespaceContainer.setSource(sourceConfig); namespaceContainer.setType(NameSpaceContainer.Type.SOURCE); folderConfig1.setName("folder1"); folderConfig1.setFullPathList(Arrays.asList(sourceConfig.getName(), folderConfig1.getName())); NameSpaceContainer folder = new NameSpaceContainer(); folder.setFolder(folderConfig1); folder.setType(NameSpaceContainer.Type.FOLDER); folderConfig2.setName("folder2"); folderConfig2.setFullPathList(Arrays.asList(sourceConfig.getName(), folderConfig2.getName())); folder = new NameSpaceContainer(); folder.setFolder(folderConfig2); folder.setType(NameSpaceContainer.Type.FOLDER);
NameSpaceContainer namespaceContainer = new NameSpaceContainer(); namespaceContainer.setSource(sourceConfig); namespaceContainer.setType(NameSpaceContainer.Type.SOURCE); folderConfig1.setName("folder1"); folderConfig1.setFullPathList(Arrays.asList(sourceConfig.getName(), folderConfig1.getName())); NameSpaceContainer folder = new NameSpaceContainer(); folder.setFolder(folderConfig1); folder.setType(NameSpaceContainer.Type.FOLDER); folderConfig2.setName("folder2"); folderConfig2.setFullPathList(Arrays.asList(sourceConfig.getName(), folderConfig2.getName())); folder = new NameSpaceContainer(); folder.setFolder(folderConfig2); folder.setType(NameSpaceContainer.Type.FOLDER); folderConfig11.setName("folder11"); folderConfig11.setFullPathList(Arrays.asList(sourceConfig.getName(), folderConfig1.getName(), folderConfig11.getName())); folder = new NameSpaceContainer(); folder.setFolder(folderConfig11); folder.setType(NameSpaceContainer.Type.FOLDER);
private DatasetConfig addDataset(IndexedStore<byte[], NameSpaceContainer> namespace, IndexedStore<DatasetSplitId, DatasetSplit> splitsStore, String id, List<String> path, int splits) { DatasetConfig ds = new DatasetConfig() .setId(new EntityId(id)) .setName(last(path)) .setFullPathList(path) .setType(DatasetType.PHYSICAL_DATASET) .setReadDefinition(new ReadDefinition().setSplitVersion(42L)); namespace.put( NamespaceServiceImpl.getKey(new NamespaceKey(path)), new NameSpaceContainer().setType(NameSpaceContainer.Type.DATASET).setFullPathList(path).setDataset(ds)); for(int i = 0; i < splits; i++) { final String splitKey = Integer.toString(i); DatasetSplit split = new DatasetSplit() .setSplitVersion(42L) .setSplitKey(splitKey); // Generate an older dataset split id DatasetSplitId splitId = UnsafeDatasetSplitIdHelper.of(ds, splitKey); splitsStore.put(splitId, split); } return ds; }