private boolean refreshSourceNames() throws NamespaceException { final Set<NamespaceKey> foundKeys = Sets.newHashSet(systemUserNamespaceService.getAllDatasets(sourceKey)); boolean changed = false; try { for (SourceTableDefinition accessor : plugin.get() .getDatasets(SYSTEM_USERNAME, msp.getDefaultRetrievalOptions())) { // names only added, never removed. Removal can be done by the full refresh (refreshSource()) if (!foundKeys.contains(accessor.getName()) && accessor.isSaveable()) { try { saver.shallowSave(accessor); changed = true; } catch (ConcurrentModificationException ex) { logger.debug("Concurrent update, ignoring.", ex); } } } // persist last refresh for across restarts. SourceInternalData srcData = sourceDataStore.get(sourceKey); if(srcData == null) { srcData = new SourceInternalData(); srcData.setName(sourceKey.getRoot()); } srcData.setLastNameRefreshDateMs(System.currentTimeMillis()); sourceDataStore.put(sourceKey, srcData); } catch (Exception ex){ logger.warn("Failure while attempting to update metadata for source {}. Terminating update of this source.", sourceKey, ex); } return changed; }
@Test public void refreshSourceMetadata_EmptySource() throws Exception { doMockDatasets(mockUpPlugin, ImmutableList.<SourceTableDefinition>of()); catalogService.refreshSource(mockUpKey, CatalogService.REFRESH_EVERYTHING_NOW, CatalogService.UpdateType.FULL); // make sure the namespace has no datasets under mockUpKey List<NamespaceKey> datasets = Lists.newArrayList(namespaceService.getAllDatasets(mockUpKey)); assertEquals(0, datasets.size()); assertNoDatasetsAfterSourceDeletion(); }
@Override public void upgrade(UpgradeContext context) throws Exception { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); try { for (SourceConfig source : namespaceService.getSources()) { if (!"HIVE".equalsIgnoreCase(ConnectionReader.toType(source))) { continue; } System.out.printf(" Handling Hive source %s%n", source.getName()); for (NamespaceKey datasetPath : namespaceService.getAllDatasets(new NamespaceKey(source.getName()))) { final DatasetConfig datasetConfig = namespaceService.getDataset(datasetPath); if (datasetConfig.getReadDefinition() == null || datasetConfig.getReadDefinition().getExtendedProperty() == null) { continue; } System.out.printf(" Clearing read definition of table %s%n", datasetPath.getSchemaPath()); datasetConfig.setReadDefinition(null); namespaceService.addOrUpdateDataset(datasetPath, datasetConfig); } } } catch (NamespaceException e) { throw new RuntimeException("Hive121BasedInputSplits failed", e); } }
private void assertNoDatasetsAfterSourceDeletion() throws Exception { final SourceConfig sourceConfig = namespaceService.getSource(mockUpKey); namespaceService.deleteSource(mockUpKey, sourceConfig.getTag()); assertEquals(0, Iterables.size(namespaceService.getAllDatasets(mockUpKey))); }
@Test public void refreshSourceMetadata_FirstTime() throws Exception { doMockDatasets(mockUpPlugin, mockDatasets); catalogService.refreshSource(mockUpKey, CatalogService.REFRESH_EVERYTHING_NOW, CatalogService.UpdateType.FULL); // make sure the namespace has datasets and folders according to the data supplied by plugin List<NamespaceKey> actualDatasetKeys = Lists.newArrayList(namespaceService.getAllDatasets(mockUpKey)); assertEquals(5, actualDatasetKeys.size()); assertDatasetsAreEqual(mockDatasets, actualDatasetKeys); assertFoldersExist(Lists.newArrayList(MOCK_UP + ".fld1", MOCK_UP + ".fld2", MOCK_UP + ".fld2.fld21")); assertDatasetSchemasDefined(actualDatasetKeys); assertNoDatasetsAfterSourceDeletion(); }
final Set<NamespaceKey> foundKeys = Sets.newHashSet(systemUserNamespaceService.getAllDatasets(sourceKey));
@Test public void refreshSourceMetadata_FirstTime_UpdateWithNewDatasets() throws Exception { doMockDatasets(mockUpPlugin, mockDatasets); catalogService.refreshSource(mockUpKey, CatalogService.REFRESH_EVERYTHING_NOW, CatalogService.UpdateType.FULL); List<NamespaceKey> actualDatasetKeys = Lists.newArrayList(namespaceService.getAllDatasets(mockUpKey)); assertEquals(5, actualDatasetKeys.size()); List<SourceTableDefinition> testDatasets = Lists.newArrayList(mockDatasets); testDatasets.add(newDataset(MOCK_UP + ".ds4")); testDatasets.add(newDataset(MOCK_UP + ".fld1.ds13")); testDatasets.add(newDataset(MOCK_UP + ".fld2.fld21.ds212")); testDatasets.add(newDataset(MOCK_UP + ".fld5.ds51")); doMockDatasets(mockUpPlugin, testDatasets); catalogService.refreshSource(mockUpKey, CatalogService.REFRESH_EVERYTHING_NOW, CatalogService.UpdateType.FULL); // make sure the namespace has datasets and folders according to the data supplied by plugin in second request actualDatasetKeys = Lists.newArrayList(namespaceService.getAllDatasets(mockUpKey)); assertEquals(9, actualDatasetKeys.size()); assertDatasetsAreEqual(testDatasets, actualDatasetKeys); assertFoldersExist(Lists.newArrayList(MOCK_UP + ".fld1", MOCK_UP + ".fld2", MOCK_UP + ".fld2.fld21", MOCK_UP + ".fld5")); assertDatasetSchemasDefined(actualDatasetKeys); assertNoDatasetsAfterSourceDeletion(); }
@Test public void refreshSourceNames() throws Exception { doMockDatasets(mockUpPlugin, mockDatasets); catalogService.refreshSource(mockUpKey, CatalogService.DEFAULT_METADATA_POLICY, CatalogService.UpdateType.NAMES); assertEquals(5, Lists.newArrayList(namespaceService.getAllDatasets(mockUpKey)).size()); List<SourceTableDefinition> testDatasets = Lists.newArrayList(mockDatasets); testDatasets.add(newDataset(MOCK_UP + ".fld1.ds13")); testDatasets.add(newDataset(MOCK_UP + ".fld2.fld21.ds212")); testDatasets.add(newDataset(MOCK_UP + ".fld2.ds23")); testDatasets.add(newDataset(MOCK_UP + ".ds4")); testDatasets.add(newDataset(MOCK_UP + ".fld5.ds51")); doMockDatasets(mockUpPlugin, testDatasets); catalogService.refreshSource(mockUpKey, CatalogService.DEFAULT_METADATA_POLICY, CatalogService.UpdateType.NAMES); // make sure the namespace has datasets and folders according to the data supplied by plugin in second request List<NamespaceKey> actualDatasetKeys = Lists.newArrayList(namespaceService.getAllDatasets(mockUpKey)); assertEquals(10, actualDatasetKeys.size()); assertDatasetsAreEqual(testDatasets, actualDatasetKeys); assertFoldersExist(Lists.newArrayList(MOCK_UP + ".fld1", MOCK_UP + ".fld2", MOCK_UP + ".fld2.fld21", MOCK_UP + ".fld5")); assertDatasetSchemasNotDefined(actualDatasetKeys); assertNoDatasetsAfterSourceDeletion(); }
@Test public void testDatasetUnderFolderOrSpace() throws Exception { try ( final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false); ) { kvstore.start(); final NamespaceService ns = new NamespaceServiceImpl(kvstore); addSpace(ns, "a"); addFolder(ns, "a.foo"); addFolder(ns, "a.foo.bar1"); addFolder(ns, "a.foo.bar2"); addFolder(ns, "a.foo.bar1.bar3"); addDS(ns, "a.ds0"); addDS(ns, "a.foo.ds1"); addDS(ns, "a.foo.ds2"); addDS(ns, "a.foo.bar1.ds3"); addDS(ns, "a.foo.bar2.ds4"); addDS(ns, "a.foo.bar1.bar3.ds5"); addDS(ns, "a.foo.bar1.bar3.ds6"); assertEquals(7, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a"))))); assertEquals(6, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a", "foo"))))); assertEquals(3, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a", "foo", "bar1"))))); assertEquals(1, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a", "foo", "bar2"))))); assertEquals(2, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a", "foo", "bar1", "bar3"))))); } }
@Test public void testDatasetsUnderHome() throws Exception { try ( final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false); ) { kvstore.start(); final NamespaceService ns = new NamespaceServiceImpl(kvstore); addHome(ns, "a"); addFolder(ns, "@a.foo"); addFolder(ns, "@a.foo.bar1"); addFolder(ns, "@a.foo.bar2"); addFolder(ns, "@a.foo.bar1.bar3"); addDS(ns, "@a.ds0"); addDS(ns, "@a.foo.ds1"); addDS(ns, "@a.foo.ds2"); addDS(ns, "@a.foo.bar1.ds3"); addDS(ns, "@a.foo.bar2.ds4"); addDS(ns, "@a.foo.bar1.bar3.ds5"); addDS(ns, "@a.foo.bar1.bar3.ds6"); assertEquals(7, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a"))))); assertEquals(6, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a", "foo"))))); assertEquals(1, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a", "foo", "bar2"))))); assertEquals(3, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a", "foo", "bar1"))))); assertEquals(2, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a", "foo", "bar1", "bar3"))))); } }
final List<NamespaceKey> allDatasets = Lists.newArrayList(namespace.getAllDatasets(key));
for (NamespaceKey datasetPath : namespaceService.getAllDatasets(new NamespaceKey(source.getName()))) { final DatasetConfig datasetConfig = namespaceService.getDataset(datasetPath);
when(mockNamespaceService.getSource(any(NamespaceKey.class))) .thenReturn(mockUpConfig); when(mockNamespaceService.getAllDatasets(any(NamespaceKey.class))) .thenReturn(Collections.emptyList());
List<NamespaceKey> actualDatasetKeys = Lists.newArrayList(namespaceService.getAllDatasets(mockUpKey)); assertEquals(5, actualDatasetKeys.size()); actualDatasetKeys = Lists.newArrayList(namespaceService.getAllDatasets(mockUpKey)); assertEquals(7, actualDatasetKeys.size());
Lists.newArrayList(namespaceService.getAllDatasets(sourceConfig.getKey()));
.getAllDatasets(new SpacePath(new SpaceName("a")).toNamespaceKey())); List<NamespaceKey> spaceB = Lists.newArrayList(service.getNamespaceService() .getAllDatasets(new SpacePath(new SpaceName("b")).toNamespaceKey())); List<NamespaceKey> spaceC = Lists.newArrayList(service.getNamespaceService() .getAllDatasets(new SpacePath(new SpaceName("c")).toNamespaceKey()));
List<NamespaceKey> tables0 = Lists.newArrayList(getSabotContext() .getNamespaceService(SystemUser.SYSTEM_USERNAME) .getAllDatasets(new NamespaceKey("hive"))); .getAllDatasets(new NamespaceKey("hive"))); assertEquals(tables0.size(), tables1.size()); .getAllDatasets(new NamespaceKey("hive"))); assertEquals(tables1.size() + 1, tables2.size()); .getAllDatasets(new NamespaceKey("hive"))); assertEquals(tables2.size(), tables3.size()); .getAllDatasets(new NamespaceKey("hive"))); assertEquals(tables3.size() - 1, tables4.size());