How to use

com.dremio.service.namespace.NamespaceServiceImpl

constructor

 @Override
 public NamespaceService get(String userName) {
  Preconditions.checkNotNull(userName, "requires userName"); // per method contract
  return new NamespaceServiceImpl(kvStoreProvider);
 }
}

private static void deleteSplitOrphans(LocalKVStoreProvider provider) {
 System.out.print("Deleting split orphans... ");
 NamespaceServiceImpl service = new NamespaceServiceImpl(provider);
 // Since the system is offline, it is okay to delete split orphans to only keep current versions.
 System.out.println(String.format("Completed. Deleted %d orphans.",
   service.deleteSplitOrphans(DatasetSplitId.SplitOrphansRetentionPolicy.KEEP_CURRENT_VERSION_ONLY)));
}

@Override
public void upgrade(UpgradeContext context) {
 namespace = new NamespaceServiceImpl(context.getKVStoreProvider());
 store = new ExternalReflectionStore(DirectProvider.wrap(context.getKVStoreProvider()));
 final Iterable<ExternalReflection> reflections = store.getExternalReflections();
 StreamSupport.stream(reflections.spliterator(), false)
  .forEach(this::update);
}

public static int pruneOrphans(LocalKVStoreProvider kvStoreProvider) {
 AtomicInteger results = new AtomicInteger();
 final NamespaceServiceImpl namespaceService = new NamespaceServiceImpl(kvStoreProvider);
 // check tags for orphans
 final CollaborationTagStore tagsStore = new CollaborationTagStore(kvStoreProvider);
 StreamSupport.stream(tagsStore.find().spliterator(), false)
  .filter(entry -> {
   // if item is not in the namespace, delete the entry
   final String entityId = entry.getValue().getEntityId();
   return namespaceService.findDatasetByUUID(entityId) == null;
  })
  .forEach(entry -> {
   results.getAndIncrement();
   tagsStore.delete(entry.getKey());
  });
 // check wikis for orphans
 final CollaborationWikiStore wikiStore = new CollaborationWikiStore(kvStoreProvider);
 StreamSupport.stream(wikiStore.find().spliterator(), false)
  .filter(entry -> {
   // if item is not in the namespace, delete the entry
   final String entityId = entry.getValue().getEntityId();
   return namespaceService.findDatasetByUUID(entityId) == null;
  })
  .forEach(entry -> {
   results.getAndIncrement();
   wikiStore.delete(entry.getKey());
  });
 return results.get();
}

 @Override
 public void upgrade(UpgradeContext context) {
  final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider());
  try {
   final NamespaceKey key = new DatasetPath(ImmutableList.of("sys", "materializations")).toNamespaceKey();
   final DatasetConfig dataset = namespaceService.getDataset(key);

   namespaceService.deleteDataset(key, dataset.getTag());
  } catch (NamespaceNotFoundException e) {
   // no metadata was found for sys.materializations
   // most likely the table was never queried
   // nothing more to do
   System.out.println("  'sys.materializations' metadata not found...skipping");
  } catch (NamespaceException e) {
   throw new RuntimeException("Failed to delete metadata for 'sys.materialization'", e);
  }
 }
}

@Bootstrap
@POST
@Path("create")
public void createTestDataset() throws Exception {
 refreshNow("cp");
 // TODO: Clean up this mess
 SampleDataPopulator.addDefaultFirstUser(userService, new NamespaceServiceImpl(provider));
 NamespaceService nsWithAuth = context.getNamespaceService(DEFAULT_USER_NAME);
 DatasetVersionMutator ds = newDS(nsWithAuth);
 // Closing sdp means remove the temporary directory
 @SuppressWarnings("resource")
 SampleDataPopulator sdp = new SampleDataPopulator(context, newSourceService(nsWithAuth, ds), ds,
   userService, nsWithAuth, DEFAULT_USER_NAME);
 sdp.populateInitialData();
}

@Override
public void upgrade(UpgradeContext context) throws Exception {
 final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider());
 List<SourceConfig> sources = namespaceService.getSources();
 for (SourceConfig sourceConfig : sources) {
  // Pre-1.5, the config object for internal sources was null.  Deleting the internal sources without a config will
  // ensure that they get recreated on startup.
  if (sourceConfig.getConfig() == null) {
   System.out.printf("  deleting '%s'%n", sourceConfig.getName());
   // following may throw an exception, we let it propagate to fail the upgrade
   namespaceService.deleteSource(sourceConfig.getKey(), sourceConfig.getTag());
  }
 }
}

private void migrateGoals(UpgradeContext context) {
 final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider());
 final Provider<KVStoreProvider> provider = DirectProvider.wrap(context.getKVStoreProvider());
 final ReflectionGoalsStore reflectionGoalsStore = new ReflectionGoalsStore(provider);
 final ReflectionEntriesStore reflectionEntriesStore = new ReflectionEntriesStore(provider);
 final MaterializationStore materializationStore = new MaterializationStore(provider);
 for (ReflectionGoal reflectionGoal : reflectionGoalsStore.getAll()) {
  try {
   migrateGoal(namespaceService, reflectionGoalsStore, reflectionEntriesStore, materializationStore, reflectionGoal);
  } catch (Exception e) {
   System.out.println(String.format("    failed migrating reflection [%s] for dataset [%s]: %s", reflectionGoal.getId(), reflectionGoal.getDatasetId(), e));
  }
 }
}

@Override
public void upgrade(UpgradeContext context) throws Exception {
 final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider());
 List<SourceConfig> sources = namespaceService.getSources();
 for (SourceConfig sourceConfig : sources) {
  ConnectionConf<?, ?> connectionConf = context.getConnectionReader().getConnectionConf(sourceConfig);
  if (connectionConf instanceof S3PluginConfig) {
   S3PluginConfig s3PluginConfig = (S3PluginConfig) connectionConf;
   if ((s3PluginConfig.credentialType == AWSAuthenticationType.ACCESS_KEY) && isNullOrEmpty(s3PluginConfig.accessKey)) {
    s3PluginConfig.credentialType = AWSAuthenticationType.NONE;
    sourceConfig.setConnectionConf(s3PluginConfig);
    namespaceService.addOrUpdateSource(sourceConfig.getKey(), sourceConfig);
   }
  }
 }
}

@Override
public void upgrade(UpgradeContext context) throws Exception {
 final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider());
 try {
  for (SourceConfig source : namespaceService.getSources()) {
   if (!"HIVE".equalsIgnoreCase(ConnectionReader.toType(source))) {
    continue;
   }
   System.out.printf("  Handling Hive source %s%n", source.getName());
   for (NamespaceKey datasetPath : namespaceService.getAllDatasets(new NamespaceKey(source.getName()))) {
    final DatasetConfig datasetConfig = namespaceService.getDataset(datasetPath);
    if (datasetConfig.getReadDefinition() == null || datasetConfig.getReadDefinition().getExtendedProperty() == null) {
     continue;
    }
    System.out.printf("    Clearing read definition of table %s%n", datasetPath.getSchemaPath());
    datasetConfig.setReadDefinition(null);
    namespaceService.addOrUpdateDataset(datasetPath, datasetConfig);
   }
  }
 } catch (NamespaceException e) {
  throw new RuntimeException("Hive121BasedInputSplits failed", e);
 }
}

@Override
public void upgrade(UpgradeContext context) {
 final NamespaceService namespace = new NamespaceServiceImpl(context.getKVStoreProvider());
 for (SourceConfig sourceConfig : namespace.getSources()) {
  MetadataPolicy metadataPolicy = sourceConfig.getMetadataPolicy();
  if (metadataPolicy != null) {
   long originalTTL = metadataPolicy.getDatasetDefinitionTtlMs();
   metadataPolicy.setDatasetDefinitionRefreshAfterMs(originalTTL);
   if (originalTTL == ORIGINAL_DEFAULT_REFRESH_MILLIS) {
    metadataPolicy.setDatasetDefinitionExpireAfterMs(CatalogService.DEFAULT_EXPIRE_MILLIS);
   } else {
    metadataPolicy.setDatasetDefinitionExpireAfterMs(3 * originalTTL);
   }
   sourceConfig.setMetadataPolicy(metadataPolicy);
   System.out.println("  Updating source " + sourceConfig.getName());
   try {
    namespace.addOrUpdateSource(new NamespaceKey(sourceConfig.getName()), sourceConfig);
   } catch (NamespaceException e) {
    Throwables.propagate(e);
   }
  }
 }
}

@Test
public void testDeleteEntityNotFound() throws Exception {
 try (final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false)) {
  kvstore.start();
  final NamespaceServiceImpl ns = new NamespaceServiceImpl(kvstore);
  try {
   ns.deleteEntity(new NamespaceKey(Arrays.asList("does", "not", "exist")), NameSpaceContainer.Type.FOLDER, "123", true);
   fail("deleteEntity should have failed.");
  } catch(NamespaceNotFoundException e) {
   // Expected
  }
 }
}

kvstore.start();
namespaceService = new NamespaceServiceImpl(kvstore);
namespaceStore = kvstore.getStore(NamespaceServiceImpl.NamespaceStoreCreator.class);
splitsStore = kvstore.getStore(NamespaceServiceImpl.DatasetSplitCreator.class);

@Test
public void testDatasetUnderFolderOrSpace() throws Exception {
 try (
   final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false);
 ) {
  kvstore.start();
  final NamespaceService ns = new NamespaceServiceImpl(kvstore);
  addSpace(ns, "a");
  addFolder(ns, "a.foo");
  addFolder(ns, "a.foo.bar1");
  addFolder(ns, "a.foo.bar2");
  addFolder(ns, "a.foo.bar1.bar3");
  addDS(ns, "a.ds0");
  addDS(ns, "a.foo.ds1");
  addDS(ns, "a.foo.ds2");
  addDS(ns, "a.foo.bar1.ds3");
  addDS(ns, "a.foo.bar2.ds4");
  addDS(ns, "a.foo.bar1.bar3.ds5");
  addDS(ns, "a.foo.bar1.bar3.ds6");
  assertEquals(7, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a")))));
  assertEquals(6, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a", "foo")))));
  assertEquals(3, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a", "foo", "bar1")))));
  assertEquals(1, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a", "foo", "bar2")))));
  assertEquals(2, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a", "foo", "bar1", "bar3")))));
 }
}

@Test
public void testDatasetsUnderHome() throws Exception {
 try (
  final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false);
 ) {
  kvstore.start();
  final NamespaceService ns = new NamespaceServiceImpl(kvstore);
  addHome(ns, "a");
  addFolder(ns, "@a.foo");
  addFolder(ns, "@a.foo.bar1");
  addFolder(ns, "@a.foo.bar2");
  addFolder(ns, "@a.foo.bar1.bar3");
  addDS(ns, "@a.ds0");
  addDS(ns, "@a.foo.ds1");
  addDS(ns, "@a.foo.ds2");
  addDS(ns, "@a.foo.bar1.ds3");
  addDS(ns, "@a.foo.bar2.ds4");
  addDS(ns, "@a.foo.bar1.bar3.ds5");
  addDS(ns, "@a.foo.bar1.bar3.ds6");
  assertEquals(7, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a")))));
  assertEquals(6, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a", "foo")))));
  assertEquals(1, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a", "foo", "bar2")))));
  assertEquals(3, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a", "foo", "bar1")))));
  assertEquals(2, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a", "foo", "bar1", "bar3")))));
 }
}

@Test
public void insertingDifferentEntityTypesAtSamePath() throws Exception {
 try (final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false)) {
  kvstore.start();
  final NamespaceService ns = new NamespaceServiceImpl(kvstore);
  addSpace(ns, "a");
  thrown.expect(ConcurrentModificationException.class);
  addSource(ns, "a");
  addFolder(ns, "a.foo");
  // Try to add dataset with path "a.foo"
  try {
   addDS(ns, "a.foo");
   fail("Expected the above call to fail");
  } catch (UserException ex) {
   assertTrue(ex.getMessage().contains("There already exists an entity of type [FOLDER] at given path [a.foo]"));
  }
  // Try to add folder with path "a.foo". There already a folder at "a.foo"
  try {
   addFolder(ns, "a.foo");
   fail("Expected the above call to fail");
  } catch (UserException ex) {
   assertTrue(ex.getMessage().contains("There already exists an entity of type [FOLDER] at given path [a.foo]"));
  }
 }
}

private void checkUpdateHelper(S3PluginConfig s3OldPluginConfig, AWSAuthenticationType authenticationType) throws Exception {
 try (final KVStoreProvider kvStoreProvider = new LocalKVStoreProvider(CLASSPATH_SCAN_RESULT, null, true, false)) {
  kvStoreProvider.start();
  KVStore<byte[], NameSpaceContainer> namespace = kvStoreProvider.getStore(NamespaceServiceImpl.NamespaceStoreCreator.class);
  newS3Source(namespace, "s3 plugin config", s3OldPluginConfig);
  // Performing upgrade
  UpdateS3CredentialType task = new UpdateS3CredentialType();
  final LogicalPlanPersistence lpPersistence = new LogicalPlanPersistence(DEFAULT_SABOT_CONFIG, CLASSPATH_SCAN_RESULT);
  final ConnectionReader connectionReader = ConnectionReader.of(CLASSPATH_SCAN_RESULT, DEFAULT_SABOT_CONFIG);
  UpgradeContext context = new UpgradeContext(kvStoreProvider, lpPersistence, connectionReader);
  task.upgrade(context);
  final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider());
  List<SourceConfig> sources = namespaceService.getSources();
  assertEquals(1, sources.size());
  ConnectionConf<?, ?> connectionConf = context.getConnectionReader().getConnectionConf(sources.get(0));
  assertTrue(connectionConf instanceof S3PluginConfig);
  S3PluginConfig s3PluginConfig = (S3PluginConfig) connectionConf;
  assertEquals(authenticationType, s3PluginConfig.credentialType);
 }
}

@Test
public void testDataSetSchema() throws Exception {
 try(
     final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false);
 ) {
  kvstore.start();
  final NamespaceService ns = new NamespaceServiceImpl(kvstore);
  Field field1 = new Field("a", true, new Int(32, true), null);
  Field child1 = new Field("c", true, Utf8.INSTANCE, null);
  Field field2 = new Field("b", true, Struct.INSTANCE, ImmutableList.of(child1));
  Schema schema = new Schema(ImmutableList.of(field1, field2));
  FlatBufferBuilder builder = new FlatBufferBuilder();
  schema.getSchema(builder);
  builder.finish(schema.getSchema(builder));
  addSource(ns, "s");
  addPhysicalDS(ns, "s.foo", builder.sizedByteArray());
  ByteBuffer bb = ByteBuffer.wrap(DatasetHelper.getSchemaBytes(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("s.foo")))).toByteArray());
  Schema returnedSchema = Schema.convertSchema(org.apache.arrow.flatbuf.Schema.getRootAsSchema(bb));
  assertEquals(schema, returnedSchema);
 }
}

@Before
public void setup() throws Exception {
 {
  SampleDataPopulator.addDefaultFirstUser(l(UserService.class), new NamespaceServiceImpl(l(KVStoreProvider.class)));
  final HDFSConf hdfsConfig = new HDFSConf();
  hdfsConfig.hostname = host;
  hdfsConfig.port = port;
  SourceConfig source = new SourceConfig();
  source.setName(SOURCE_NAME);
  source.setMetadataPolicy(CatalogService.DEFAULT_METADATA_POLICY_WITH_AUTO_PROMOTE);
  source.setConnectionConf(hdfsConfig);
  source.setId(new EntityId(SOURCE_ID));
  source.setDescription(SOURCE_DESC);
  ((CatalogServiceImpl)l(CatalogService.class)).getSystemUserCatalog().createSource(source);
 }
}

@Test
public void testGetDatasetCount() throws Exception {
 try (final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false)) {
  kvstore.start();
  final NamespaceService ns = new NamespaceServiceImpl(kvstore);
  // create some nested datasets
  addSource(ns, "a");
  for (int i = 0; i < 10; i++) {
   addDS(ns, "a.foo" + i);
  }
  for (int i = 0; i < 50; i++) {
   addDS(ns, "a.foo0.bar" + i);
  }
  for (int i = 0; i < 50; i++) {
   addDS(ns, "a.baz" + i);
  }
  // test count bound
  BoundedDatasetCount boundedDatasetCount = ns.getDatasetCount(new NamespaceKey("a"), 5000, 30);
  assertTrue(boundedDatasetCount.isCountBound());
  assertEquals(boundedDatasetCount.getCount(), 30);
  // test time bound - the code checks every 50 children visited to see if the time bound has been hit and we give it 0 ms
  boundedDatasetCount = ns.getDatasetCount(new NamespaceKey("a"), 0, 1000);
  assertTrue(boundedDatasetCount.isTimeBound());
 }
}