com.dremio.service.namespace.NamespaceServiceImpl.<init> java code examples

Common ways to obtain NamespaceServiceImpl

private void myMethod () {
 NamespaceServiceImpl n =KVStoreProvider kvStoreProvider;new NamespaceServiceImpl(kvStoreProvider)
Smart code suggestions by Tabnine
}

 @Override
 public NamespaceService get(String userName) {
  Preconditions.checkNotNull(userName, "requires userName"); // per method contract
  return new NamespaceServiceImpl(kvStoreProvider);
 }
}

private static void deleteSplitOrphans(LocalKVStoreProvider provider) {
 System.out.print("Deleting split orphans... ");
 NamespaceServiceImpl service = new NamespaceServiceImpl(provider);
 // Since the system is offline, it is okay to delete split orphans to only keep current versions.
 System.out.println(String.format("Completed. Deleted %d orphans.",
   service.deleteSplitOrphans(DatasetSplitId.SplitOrphansRetentionPolicy.KEEP_CURRENT_VERSION_ONLY)));
}

@Override
public void upgrade(UpgradeContext context) {
 namespace = new NamespaceServiceImpl(context.getKVStoreProvider());
 store = new ExternalReflectionStore(DirectProvider.wrap(context.getKVStoreProvider()));
 final Iterable<ExternalReflection> reflections = store.getExternalReflections();
 StreamSupport.stream(reflections.spliterator(), false)
  .forEach(this::update);
}

public static int pruneOrphans(LocalKVStoreProvider kvStoreProvider) {
 AtomicInteger results = new AtomicInteger();
 final NamespaceServiceImpl namespaceService = new NamespaceServiceImpl(kvStoreProvider);
 // check tags for orphans
 final CollaborationTagStore tagsStore = new CollaborationTagStore(kvStoreProvider);
 StreamSupport.stream(tagsStore.find().spliterator(), false)
  .filter(entry -> {
   // if item is not in the namespace, delete the entry
   final String entityId = entry.getValue().getEntityId();
   return namespaceService.findDatasetByUUID(entityId) == null;
  })
  .forEach(entry -> {
   results.getAndIncrement();
   tagsStore.delete(entry.getKey());
  });
 // check wikis for orphans
 final CollaborationWikiStore wikiStore = new CollaborationWikiStore(kvStoreProvider);
 StreamSupport.stream(wikiStore.find().spliterator(), false)
  .filter(entry -> {
   // if item is not in the namespace, delete the entry
   final String entityId = entry.getValue().getEntityId();
   return namespaceService.findDatasetByUUID(entityId) == null;
  })
  .forEach(entry -> {
   results.getAndIncrement();
   wikiStore.delete(entry.getKey());
  });
 return results.get();
}

 @Override
 public void upgrade(UpgradeContext context) {
  final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider());
  try {
   final NamespaceKey key = new DatasetPath(ImmutableList.of("sys", "materializations")).toNamespaceKey();
   final DatasetConfig dataset = namespaceService.getDataset(key);

   namespaceService.deleteDataset(key, dataset.getTag());
  } catch (NamespaceNotFoundException e) {
   // no metadata was found for sys.materializations
   // most likely the table was never queried
   // nothing more to do
   System.out.println("  'sys.materializations' metadata not found...skipping");
  } catch (NamespaceException e) {
   throw new RuntimeException("Failed to delete metadata for 'sys.materialization'", e);
  }
 }
}

@Bootstrap
@POST
@Path("create")
public void createTestDataset() throws Exception {
 refreshNow("cp");
 // TODO: Clean up this mess
 SampleDataPopulator.addDefaultFirstUser(userService, new NamespaceServiceImpl(provider));
 NamespaceService nsWithAuth = context.getNamespaceService(DEFAULT_USER_NAME);
 DatasetVersionMutator ds = newDS(nsWithAuth);
 // Closing sdp means remove the temporary directory
 @SuppressWarnings("resource")
 SampleDataPopulator sdp = new SampleDataPopulator(context, newSourceService(nsWithAuth, ds), ds,
   userService, nsWithAuth, DEFAULT_USER_NAME);
 sdp.populateInitialData();
}

@Override
public void upgrade(UpgradeContext context) throws Exception {
 final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider());
 List<SourceConfig> sources = namespaceService.getSources();
 for (SourceConfig sourceConfig : sources) {
  // Pre-1.5, the config object for internal sources was null.  Deleting the internal sources without a config will
  // ensure that they get recreated on startup.
  if (sourceConfig.getConfig() == null) {
   System.out.printf("  deleting '%s'%n", sourceConfig.getName());
   // following may throw an exception, we let it propagate to fail the upgrade
   namespaceService.deleteSource(sourceConfig.getKey(), sourceConfig.getTag());
  }
 }
}

private void migrateGoals(UpgradeContext context) {
 final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider());
 final Provider<KVStoreProvider> provider = DirectProvider.wrap(context.getKVStoreProvider());
 final ReflectionGoalsStore reflectionGoalsStore = new ReflectionGoalsStore(provider);
 final ReflectionEntriesStore reflectionEntriesStore = new ReflectionEntriesStore(provider);
 final MaterializationStore materializationStore = new MaterializationStore(provider);
 for (ReflectionGoal reflectionGoal : reflectionGoalsStore.getAll()) {
  try {
   migrateGoal(namespaceService, reflectionGoalsStore, reflectionEntriesStore, materializationStore, reflectionGoal);
  } catch (Exception e) {
   System.out.println(String.format("    failed migrating reflection [%s] for dataset [%s]: %s", reflectionGoal.getId(), reflectionGoal.getDatasetId(), e));
  }
 }
}

@Override
public void upgrade(UpgradeContext context) throws Exception {
 final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider());
 List<SourceConfig> sources = namespaceService.getSources();
 for (SourceConfig sourceConfig : sources) {
  ConnectionConf<?, ?> connectionConf = context.getConnectionReader().getConnectionConf(sourceConfig);
  if (connectionConf instanceof S3PluginConfig) {
   S3PluginConfig s3PluginConfig = (S3PluginConfig) connectionConf;
   if ((s3PluginConfig.credentialType == AWSAuthenticationType.ACCESS_KEY) && isNullOrEmpty(s3PluginConfig.accessKey)) {
    s3PluginConfig.credentialType = AWSAuthenticationType.NONE;
    sourceConfig.setConnectionConf(s3PluginConfig);
    namespaceService.addOrUpdateSource(sourceConfig.getKey(), sourceConfig);
   }
  }
 }
}

@Override
public void upgrade(UpgradeContext context) throws Exception {
 final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider());
 try {
  for (SourceConfig source : namespaceService.getSources()) {
   if (!"HIVE".equalsIgnoreCase(ConnectionReader.toType(source))) {
    continue;
   }
   System.out.printf("  Handling Hive source %s%n", source.getName());
   for (NamespaceKey datasetPath : namespaceService.getAllDatasets(new NamespaceKey(source.getName()))) {
    final DatasetConfig datasetConfig = namespaceService.getDataset(datasetPath);
    if (datasetConfig.getReadDefinition() == null || datasetConfig.getReadDefinition().getExtendedProperty() == null) {
     continue;
    }
    System.out.printf("    Clearing read definition of table %s%n", datasetPath.getSchemaPath());
    datasetConfig.setReadDefinition(null);
    namespaceService.addOrUpdateDataset(datasetPath, datasetConfig);
   }
  }
 } catch (NamespaceException e) {
  throw new RuntimeException("Hive121BasedInputSplits failed", e);
 }
}

@Override
public void upgrade(UpgradeContext context) {
 final NamespaceService namespace = new NamespaceServiceImpl(context.getKVStoreProvider());
 for (SourceConfig sourceConfig : namespace.getSources()) {
  MetadataPolicy metadataPolicy = sourceConfig.getMetadataPolicy();
  if (metadataPolicy != null) {
   long originalTTL = metadataPolicy.getDatasetDefinitionTtlMs();
   metadataPolicy.setDatasetDefinitionRefreshAfterMs(originalTTL);
   if (originalTTL == ORIGINAL_DEFAULT_REFRESH_MILLIS) {
    metadataPolicy.setDatasetDefinitionExpireAfterMs(CatalogService.DEFAULT_EXPIRE_MILLIS);
   } else {
    metadataPolicy.setDatasetDefinitionExpireAfterMs(3 * originalTTL);
   }
   sourceConfig.setMetadataPolicy(metadataPolicy);
   System.out.println("  Updating source " + sourceConfig.getName());
   try {
    namespace.addOrUpdateSource(new NamespaceKey(sourceConfig.getName()), sourceConfig);
   } catch (NamespaceException e) {
    Throwables.propagate(e);
   }
  }
 }
}

@Test
public void testDeleteEntityNotFound() throws Exception {
 try (final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false)) {
  kvstore.start();
  final NamespaceServiceImpl ns = new NamespaceServiceImpl(kvstore);
  try {
   ns.deleteEntity(new NamespaceKey(Arrays.asList("does", "not", "exist")), NameSpaceContainer.Type.FOLDER, "123", true);
   fail("deleteEntity should have failed.");
  } catch(NamespaceNotFoundException e) {
   // Expected
  }
 }
}

kvstore.start();
namespaceService = new NamespaceServiceImpl(kvstore);
namespaceStore = kvstore.getStore(NamespaceServiceImpl.NamespaceStoreCreator.class);
splitsStore = kvstore.getStore(NamespaceServiceImpl.DatasetSplitCreator.class);

@Test
public void testDatasetUnderFolderOrSpace() throws Exception {
 try (
   final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false);
 ) {
  kvstore.start();
  final NamespaceService ns = new NamespaceServiceImpl(kvstore);
  addSpace(ns, "a");
  addFolder(ns, "a.foo");
  addFolder(ns, "a.foo.bar1");
  addFolder(ns, "a.foo.bar2");
  addFolder(ns, "a.foo.bar1.bar3");
  addDS(ns, "a.ds0");
  addDS(ns, "a.foo.ds1");
  addDS(ns, "a.foo.ds2");
  addDS(ns, "a.foo.bar1.ds3");
  addDS(ns, "a.foo.bar2.ds4");
  addDS(ns, "a.foo.bar1.bar3.ds5");
  addDS(ns, "a.foo.bar1.bar3.ds6");
  assertEquals(7, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a")))));
  assertEquals(6, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a", "foo")))));
  assertEquals(3, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a", "foo", "bar1")))));
  assertEquals(1, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a", "foo", "bar2")))));
  assertEquals(2, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a", "foo", "bar1", "bar3")))));
 }
}

@Test
public void testDatasetsUnderHome() throws Exception {
 try (
  final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false);
 ) {
  kvstore.start();
  final NamespaceService ns = new NamespaceServiceImpl(kvstore);
  addHome(ns, "a");
  addFolder(ns, "@a.foo");
  addFolder(ns, "@a.foo.bar1");
  addFolder(ns, "@a.foo.bar2");
  addFolder(ns, "@a.foo.bar1.bar3");
  addDS(ns, "@a.ds0");
  addDS(ns, "@a.foo.ds1");
  addDS(ns, "@a.foo.ds2");
  addDS(ns, "@a.foo.bar1.ds3");
  addDS(ns, "@a.foo.bar2.ds4");
  addDS(ns, "@a.foo.bar1.bar3.ds5");
  addDS(ns, "@a.foo.bar1.bar3.ds6");
  assertEquals(7, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a")))));
  assertEquals(6, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a", "foo")))));
  assertEquals(1, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a", "foo", "bar2")))));
  assertEquals(3, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a", "foo", "bar1")))));
  assertEquals(2, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a", "foo", "bar1", "bar3")))));
 }
}

@Test
public void insertingDifferentEntityTypesAtSamePath() throws Exception {
 try (final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false)) {
  kvstore.start();
  final NamespaceService ns = new NamespaceServiceImpl(kvstore);
  addSpace(ns, "a");
  thrown.expect(ConcurrentModificationException.class);
  addSource(ns, "a");
  addFolder(ns, "a.foo");
  // Try to add dataset with path "a.foo"
  try {
   addDS(ns, "a.foo");
   fail("Expected the above call to fail");
  } catch (UserException ex) {
   assertTrue(ex.getMessage().contains("There already exists an entity of type [FOLDER] at given path [a.foo]"));
  }
  // Try to add folder with path "a.foo". There already a folder at "a.foo"
  try {
   addFolder(ns, "a.foo");
   fail("Expected the above call to fail");
  } catch (UserException ex) {
   assertTrue(ex.getMessage().contains("There already exists an entity of type [FOLDER] at given path [a.foo]"));
  }
 }
}

private void checkUpdateHelper(S3PluginConfig s3OldPluginConfig, AWSAuthenticationType authenticationType) throws Exception {
 try (final KVStoreProvider kvStoreProvider = new LocalKVStoreProvider(CLASSPATH_SCAN_RESULT, null, true, false)) {
  kvStoreProvider.start();
  KVStore<byte[], NameSpaceContainer> namespace = kvStoreProvider.getStore(NamespaceServiceImpl.NamespaceStoreCreator.class);
  newS3Source(namespace, "s3 plugin config", s3OldPluginConfig);
  // Performing upgrade
  UpdateS3CredentialType task = new UpdateS3CredentialType();
  final LogicalPlanPersistence lpPersistence = new LogicalPlanPersistence(DEFAULT_SABOT_CONFIG, CLASSPATH_SCAN_RESULT);
  final ConnectionReader connectionReader = ConnectionReader.of(CLASSPATH_SCAN_RESULT, DEFAULT_SABOT_CONFIG);
  UpgradeContext context = new UpgradeContext(kvStoreProvider, lpPersistence, connectionReader);
  task.upgrade(context);
  final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider());
  List<SourceConfig> sources = namespaceService.getSources();
  assertEquals(1, sources.size());
  ConnectionConf<?, ?> connectionConf = context.getConnectionReader().getConnectionConf(sources.get(0));
  assertTrue(connectionConf instanceof S3PluginConfig);
  S3PluginConfig s3PluginConfig = (S3PluginConfig) connectionConf;
  assertEquals(authenticationType, s3PluginConfig.credentialType);
 }
}

@Test
public void testDataSetSchema() throws Exception {
 try(
     final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false);
 ) {
  kvstore.start();
  final NamespaceService ns = new NamespaceServiceImpl(kvstore);
  Field field1 = new Field("a", true, new Int(32, true), null);
  Field child1 = new Field("c", true, Utf8.INSTANCE, null);
  Field field2 = new Field("b", true, Struct.INSTANCE, ImmutableList.of(child1));
  Schema schema = new Schema(ImmutableList.of(field1, field2));
  FlatBufferBuilder builder = new FlatBufferBuilder();
  schema.getSchema(builder);
  builder.finish(schema.getSchema(builder));
  addSource(ns, "s");
  addPhysicalDS(ns, "s.foo", builder.sizedByteArray());
  ByteBuffer bb = ByteBuffer.wrap(DatasetHelper.getSchemaBytes(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("s.foo")))).toByteArray());
  Schema returnedSchema = Schema.convertSchema(org.apache.arrow.flatbuf.Schema.getRootAsSchema(bb));
  assertEquals(schema, returnedSchema);
 }
}

@Before
public void setup() throws Exception {
 {
  SampleDataPopulator.addDefaultFirstUser(l(UserService.class), new NamespaceServiceImpl(l(KVStoreProvider.class)));
  final HDFSConf hdfsConfig = new HDFSConf();
  hdfsConfig.hostname = host;
  hdfsConfig.port = port;
  SourceConfig source = new SourceConfig();
  source.setName(SOURCE_NAME);
  source.setMetadataPolicy(CatalogService.DEFAULT_METADATA_POLICY_WITH_AUTO_PROMOTE);
  source.setConnectionConf(hdfsConfig);
  source.setId(new EntityId(SOURCE_ID));
  source.setDescription(SOURCE_DESC);
  ((CatalogServiceImpl)l(CatalogService.class)).getSystemUserCatalog().createSource(source);
 }
}

@Test
public void testGetDatasetCount() throws Exception {
 try (final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false)) {
  kvstore.start();
  final NamespaceService ns = new NamespaceServiceImpl(kvstore);
  // create some nested datasets
  addSource(ns, "a");
  for (int i = 0; i < 10; i++) {
   addDS(ns, "a.foo" + i);
  }
  for (int i = 0; i < 50; i++) {
   addDS(ns, "a.foo0.bar" + i);
  }
  for (int i = 0; i < 50; i++) {
   addDS(ns, "a.baz" + i);
  }
  // test count bound
  BoundedDatasetCount boundedDatasetCount = ns.getDatasetCount(new NamespaceKey("a"), 5000, 30);
  assertTrue(boundedDatasetCount.isCountBound());
  assertEquals(boundedDatasetCount.getCount(), 30);
  // test time bound - the code checks every 50 children visited to see if the time bound has been hit and we give it 0 ms
  boundedDatasetCount = ns.getDatasetCount(new NamespaceKey("a"), 0, 1000);
  assertTrue(boundedDatasetCount.isTimeBound());
 }
}

Popular methods of NamespaceServiceImpl

getKey
compareSplits
Return true if new splits are not same as old splits
deleteEntity
addOrUpdateDataset
addOrUpdateFolder
createOrUpdateEntity
Helper method which creates a new entity or update the existing entity with given entity
createSourceFolders
deleteSplitOrphans
deleteSplits
doCreateOrUpdateEntity
doDeleteEntity
doGetEntities

Popular in Java

Reactive rest calls using spring rest template
runOnUiThread (Activity)
orElseThrow (Optional)
Return the contained value, if present, otherwise throw an exception to be created by the provided s
requestLocationUpdates (LocationManager)
Graphics2D (java.awt)
This Graphics2D class extends the Graphics class to provide more sophisticated control overgraphics
ServerSocket (java.net)
This class represents a server-side socket that waits for incoming client connections. A ServerSocke
ByteBuffer (java.nio)
A buffer for bytes. A byte buffer can be created in either one of the following ways: * #allocate
Hashtable (java.util)
A plug-in replacement for JDK1.5 java.util.Hashtable. This version is based on org.cliffc.high_scale
SortedSet (java.util)
SortedSet is a Set which iterates over its elements in a sorted order. The order is determined eithe
TimerTask (java.util)
The TimerTask class represents a task to run at a specified time. The task may be run once or repeat
Top 25 Plugins for Webstorm

How to use com.dremio.service.namespace.NamespaceServiceImplconstructor

Best Java code snippets using com.dremio.service.namespace.NamespaceServiceImpl.<init> (Showing top 20 results out of 315)

How to use
com.dremio.service.namespace.NamespaceServiceImpl
constructor