@Override public List<SourceConfig> getSources(String username) { return factoryProvider.get() .get(username) .getSources(); }
public List<SourceConfig> getSources() { final List<SourceConfig> sources = new ArrayList<>(); for (SourceConfig sourceConfig : namespaceService.getSources()) { if (SourceUI.isInternal(sourceConfig, connectionReader)) { continue; } sources.add(sourceConfig); } return sources; }
/** * Delete all sources except those that are specifically referenced * @param rootsToSaveSet Root sources to save. * @throws NamespaceException */ @VisibleForTesting public void deleteExcept(Set<String> rootsToSaveSet) throws NamespaceException { for(SourceConfig source : systemNamespace.getSources()) { if(rootsToSaveSet.contains(source.getName())) { continue; } deleteSource(source, SystemUser.SYSTEM_USERNAME); } }
for(SourceConfig config : systemNamespace.getSources()) { names.remove(config.getName()); try {
@Override public void upgrade(UpgradeContext context) throws Exception { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); List<SourceConfig> sources = namespaceService.getSources(); for (SourceConfig sourceConfig : sources) { // Pre-1.5, the config object for internal sources was null. Deleting the internal sources without a config will // ensure that they get recreated on startup. if (sourceConfig.getConfig() == null) { System.out.printf(" deleting '%s'%n", sourceConfig.getName()); // following may throw an exception, we let it propagate to fail the upgrade namespaceService.deleteSource(sourceConfig.getKey(), sourceConfig.getTag()); } } }
@Override public void upgrade(UpgradeContext context) throws Exception { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); List<SourceConfig> sources = namespaceService.getSources(); for (SourceConfig sourceConfig : sources) { ConnectionConf<?, ?> connectionConf = context.getConnectionReader().getConnectionConf(sourceConfig); if (connectionConf instanceof S3PluginConfig) { S3PluginConfig s3PluginConfig = (S3PluginConfig) connectionConf; if ((s3PluginConfig.credentialType == AWSAuthenticationType.ACCESS_KEY) && isNullOrEmpty(s3PluginConfig.accessKey)) { s3PluginConfig.credentialType = AWSAuthenticationType.NONE; sourceConfig.setConnectionConf(s3PluginConfig); namespaceService.addOrUpdateSource(sourceConfig.getKey(), sourceConfig); } } } }
@Override public void upgrade(UpgradeContext context) throws Exception { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); try { for (SourceConfig source : namespaceService.getSources()) { if (!"HIVE".equalsIgnoreCase(ConnectionReader.toType(source))) { continue; } System.out.printf(" Handling Hive source %s%n", source.getName()); for (NamespaceKey datasetPath : namespaceService.getAllDatasets(new NamespaceKey(source.getName()))) { final DatasetConfig datasetConfig = namespaceService.getDataset(datasetPath); if (datasetConfig.getReadDefinition() == null || datasetConfig.getReadDefinition().getExtendedProperty() == null) { continue; } System.out.printf(" Clearing read definition of table %s%n", datasetPath.getSchemaPath()); datasetConfig.setReadDefinition(null); namespaceService.addOrUpdateDataset(datasetPath, datasetConfig); } } } catch (NamespaceException e) { throw new RuntimeException("Hive121BasedInputSplits failed", e); } }
private ClusterInfo getClusterInfo(){ SoftwareVersion version = new SoftwareVersion().setVersion(DremioVersionInfo.getVersion()); List<Source> sources = new ArrayList<>(); final NamespaceService ns = executionContextProvider.get().getNamespaceService(SystemUser.SYSTEM_USERNAME); for(SourceConfig source : ns.getSources()){ String type = source.getType() == null ? source.getLegacySourceTypeEnum().name() : source.getType(); sources.add(new Source().setName(source.getName()).setType(type)); } List<Node> nodes = new ArrayList<>(); for(NodeEndpoint ep : executionContextProvider.get().getExecutors()){ nodes.add(new Node().setName(ep.getAddress()).setRole("executor")); } for(NodeEndpoint ep : executionContextProvider.get().getCoordinators()){ nodes.add(new Node().setName(ep.getAddress()).setRole("coordinator")); } return new ClusterInfo() .setIdentity(identity) .setVersion(version) .setSourceList(sources) .setNodeList(nodes) .setJavaVmVersion(System.getProperty("java.vm.version")) .setJreVersion(System.getProperty("java.specification.version")) .setEdition(getEditionInfo()) ; }
@Override public void upgrade(UpgradeContext context) { final NamespaceService namespace = new NamespaceServiceImpl(context.getKVStoreProvider()); for (SourceConfig sourceConfig : namespace.getSources()) { MetadataPolicy metadataPolicy = sourceConfig.getMetadataPolicy(); if (metadataPolicy != null) { long originalTTL = metadataPolicy.getDatasetDefinitionTtlMs(); metadataPolicy.setDatasetDefinitionRefreshAfterMs(originalTTL); if (originalTTL == ORIGINAL_DEFAULT_REFRESH_MILLIS) { metadataPolicy.setDatasetDefinitionExpireAfterMs(CatalogService.DEFAULT_EXPIRE_MILLIS); } else { metadataPolicy.setDatasetDefinitionExpireAfterMs(3 * originalTTL); } sourceConfig.setMetadataPolicy(metadataPolicy); System.out.println(" Updating source " + sourceConfig.getName()); try { namespace.addOrUpdateSource(new NamespaceKey(sourceConfig.getName()), sourceConfig); } catch (NamespaceException e) { Throwables.propagate(e); } } } }
private CheckPoint checkPoint() throws Exception { CheckPoint checkPoint = new CheckPoint(); NamespaceService namespaceService = newNamespaceService(); UserService userService = l(UserService.class); JobsService jobsService = l(JobsService.class); checkPoint.sources = namespaceService.getSources(); checkPoint.spaces = namespaceService.getSpaces(); checkPoint.homes = namespaceService.getHomeSpaces(); checkPoint.users = Arrays.asList(Iterables.toArray(userService.getAllUsers(10000), User.class)); checkPoint.datasets = Lists.newArrayList(); checkPoint.virtualDatasetVersions = Lists.newArrayList(); checkPoint.accelerations = Lists.newArrayList(); /** DX-4498 for (NamespaceKey ds : namespaceService.getAllDatasets(new NamespaceKey(""))) { DatasetConfig datasetConfig = namespaceService.getDataset(ds); checkPoint.datasets.add(datasetConfig); checkPoint.virtualDatasetVersions.addAll( Arrays.asList(Iterables.toArray(datasetService.getAllVersions(new DatasetPath(ds.getPathComponents())), VirtualDatasetUI.class))); if (datasetConfig.getAccelerationId() != null) { checkPoint.accelerations.add(accelerationService.getAccelerationById(new AccelerationId(datasetConfig.getAccelerationId()))); } } */ checkPoint.jobs = ImmutableList.copyOf(jobsService.getAllJobs(null, null, null, 0, Integer.MAX_VALUE, "tshiran")); return checkPoint; }
final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); try { for (SourceConfig source : namespaceService.getSources()) { if (!"HIVE".equalsIgnoreCase(ConnectionReader.toType(source))) { continue;
.thenReturn(mockUpConfig); when(mockNamespaceService.getSources()) .thenReturn(Arrays.asList(mockUpConfig)); when(mockNamespaceService.getSource(any(NamespaceKey.class)))
DirectProvider.wrap(context.getKVStoreProvider())); List<SourceConfig> sources = namespaceService.getSources(); for (SourceConfig sourceConfig : sources) { try {
boolean needToEnableSubHourPolicies = false; for (SourceConfig sourceConfig : namespace.getSources()) { NamespaceKey sourceKey = new NamespaceKey(sourceConfig.getName()); TimePeriod accelerationTTL = sourceConfig.getAccelerationTTL();
List<SourceConfig> sources = ns.getSources(); System.out.println(sources); assertEquals(3, sources.size());
private void checkUpdateHelper(S3PluginConfig s3OldPluginConfig, AWSAuthenticationType authenticationType) throws Exception { try (final KVStoreProvider kvStoreProvider = new LocalKVStoreProvider(CLASSPATH_SCAN_RESULT, null, true, false)) { kvStoreProvider.start(); KVStore<byte[], NameSpaceContainer> namespace = kvStoreProvider.getStore(NamespaceServiceImpl.NamespaceStoreCreator.class); newS3Source(namespace, "s3 plugin config", s3OldPluginConfig); // Performing upgrade UpdateS3CredentialType task = new UpdateS3CredentialType(); final LogicalPlanPersistence lpPersistence = new LogicalPlanPersistence(DEFAULT_SABOT_CONFIG, CLASSPATH_SCAN_RESULT); final ConnectionReader connectionReader = ConnectionReader.of(CLASSPATH_SCAN_RESULT, DEFAULT_SABOT_CONFIG); UpgradeContext context = new UpgradeContext(kvStoreProvider, lpPersistence, connectionReader); task.upgrade(context); final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); List<SourceConfig> sources = namespaceService.getSources(); assertEquals(1, sources.size()); ConnectionConf<?, ?> connectionConf = context.getConnectionReader().getConnectionConf(sources.get(0)); assertTrue(connectionConf instanceof S3PluginConfig); S3PluginConfig s3PluginConfig = (S3PluginConfig) connectionConf; assertEquals(authenticationType, s3PluginConfig.credentialType); } }