@Override public String toString() { return "fix-fail-to-start-" + sourceKey.getRoot(); } }
@Override public String toString() { return "metadata-refresh-" + sourceKey.getRoot(); } }
@Override public String toString() { return "Plugin Startup: " + sourceKey.getRoot(); }
@Override public String apply(ManagedStoragePlugin input) { return input.getName().getRoot(); }}).toSet()); }
@Override public String toString() { return "late-load-" + plugin.getName().getRoot(); } }
@Override public TableType getJdbcTableType() { // ugly way to return correct table type for the system tables and information schema. if(dataset.getName().getRoot().equals("sys") || dataset.getName().getRoot().equals("INFORMATION_SCHEMA") ) { return TableType.SYSTEM_TABLE; } return TableType.TABLE; }
/** * A path is ambiguous if it has two parts and the root contains a period. In this case, we don't * know whether the root should be considered a single part of many parts and need to do a search * for an unescaped path rather than a lookup. * * This is because JDBC & ODBC tools use a two part naming scheme and thus we also present Dremio * datasets using this two part scheme where all parts of the path except the leaf are presented * as part of the schema of the table. This relates to DX- * * @param key * Key to test * @return Whether path is ambiguous. */ private boolean isAmbiguousKey(NamespaceKey key) { if(key.size() != 2) { return false; } return key.getRoot().contains("."); }
private FileSystemPlugin asFSn(NamespaceKey key) { final StoragePlugin plugin = context.getCatalogService().getSource(key.getRoot()); if (plugin instanceof FileSystemPlugin) { return (FileSystemPlugin) plugin; } else { return null; } }
@Override public UpdateStatus refreshDataset(NamespaceKey key, DatasetRetrievalOptions retrievalOptions) { ManagedStoragePlugin plugin = pluginRetriever.getPlugin(key.getRoot(), true); if(plugin == null){ throw UserException.validationError().message("Unknown source %s", key.getRoot()).build(logger); } return plugin.refreshDataset(key, retrievalOptions); }
@Override public void createDataset(NamespaceKey key, com.google.common.base.Function<DatasetConfig, DatasetConfig> datasetMutator) { ManagedStoragePlugin plugin = pluginRetriever.getPlugin(key.getRoot(), true); if(plugin == null){ throw UserException.validationError().message("Unknown source %s", key.getRoot()).build(logger); } datasets.createDataset(key, plugin, datasetMutator); }
@Override public boolean refreshSource(NamespaceKey source, MetadataPolicy metadataPolicy, UpdateType updateType) throws NamespaceException { ManagedStoragePlugin plugin = plugins.get(source.getRoot()); if(plugin == null){ throw UserException.validationError().message("Unknown source %s", source.getRoot()).build(logger); } return plugin.refresh(updateType, metadataPolicy); }
@Override public boolean createOrUpdateDataset(NamespaceService userNamespaceService, NamespaceKey source, NamespaceKey datasetPath, DatasetConfig datasetConfig, NamespaceAttribute... attributes) throws NamespaceException { ManagedStoragePlugin plugin = pluginRetriever.getPlugin(source.getRoot(), true); if(plugin == null){ throw UserException.validationError().message("Unknown source %s", datasetPath.getRoot()).build(logger); } return datasets.createOrUpdateDataset(userNamespaceService, plugin, source, datasetPath, datasetConfig, attributes); }
@Override public Iterable<String> getSubPartitions( NamespaceKey key, List<String> partitionColumns, List<String> partitionValues) throws PartitionNotFoundException { if(pluginRetriever.getPlugin(key.getRoot(), true) == null){ throw UserException.validationError().message("Unknown source %s", key.getRoot()).build(logger); } StoragePlugin plugin = context.getCatalogService().getSource(key.getRoot()); FileSystemPlugin fsPlugin; if (plugin instanceof FileSystemPlugin) { fsPlugin = (FileSystemPlugin) plugin; return fsPlugin.getSubPartitions(key.getPathComponents(), partitionColumns, partitionValues, options.getSchemaConfig()); } throw new UnsupportedOperationException(plugin.getClass().getName() + " does not support partition retrieval."); }
private void addSourcesAndOrigins(DocumentWriter writer, List<FieldOrigin> fieldOrigins) { if (notEmpty(fieldOrigins)) { Set<String> sources = Sets.newHashSet(); final List<String> empty = ImmutableList.of(); for (FieldOrigin fieldOrigin : fieldOrigins) { for (Origin origin : listNotNull(fieldOrigin.getOriginsList())) { // DX-3999: fix this in calcite final List<String> path = Optional.fromNullable(origin.getTableList()).or(empty); if (path.isEmpty()) { continue; } NamespaceKey dataset = new NamespaceKey(origin.getTableList()); sources.add(dataset.getRoot()); } } writer.write(DATASET_SOURCES, sources.toArray(new String[0])); } }
@Override public SourceState refreshSourceStatus(NamespaceKey key) throws Exception { ManagedStoragePlugin plugin = pluginRetriever.getPlugin(key.getRoot(), true); if(plugin == null){ throw UserException.validationError().message("Unknown source %s", key.getRoot()).build(logger); } plugin.refreshState(); plugin.initiateMetadataRefresh(); return plugin.getState(); }
private final FileSystemPlugin getPlugin(NamespaceKey key) { StoragePlugin plugin = catalogService.getSource(key.getRoot()); if(plugin instanceof FileSystemPlugin) { return (FileSystemPlugin) plugin; } else { throw UserException.validationError() .message("Source identified was invalid type. Only sources that can contain files or folders can detect a format.") .build(logger); } }
@Override public List<SimpleCommandResult> toResult(String sql, SqlNode sqlNode) throws Exception { final SqlUseSchema useSchema = SqlNodeUtil.unwrap(sqlNode, SqlUseSchema.class); // first we try locally. NamespaceKey orig = new NamespaceKey(useSchema.getSchema()); NamespaceKey defaultPath = catalog.resolveToDefault(orig); NamespaceKey compoundPath = orig.size() == 1 && orig.getRoot().contains(".") ? new NamespaceKey(SqlUtils.parseSchemaPath(orig.getRoot())) : null; if(defaultPath != null && catalog.containerExists(defaultPath)) { session.setDefaultSchemaPath(defaultPath.getPathComponents()); } else if (catalog.containerExists(orig)) { session.setDefaultSchemaPath(orig.getPathComponents()); } else if(compoundPath != null && catalog.containerExists(compoundPath)) { // kept to support old compound use statements. session.setDefaultSchemaPath(compoundPath.getPathComponents()); } else { throw UserException.validationError().message("Schema [%s] is not valid with respect to either root schema or current default schema.", orig).build(logger); } return Collections.singletonList(SimpleCommandResult.successful("Default schema changed to [%s]", session.getDefaultSchemaPath())); } }
@Test public void testGetWiki() throws Exception { // create space NamespaceKey spacePath = new NamespaceKey("testspace"); List<String> vdsPath = Arrays.asList(spacePath.getRoot(), "testVDS"); createSpaceAndVDS(spacePath, vdsPath); DatasetConfig dataset = newNamespaceService().getDataset(new NamespaceKey(vdsPath)); // no tags initially, so expect a 404 expectStatus(NOT_FOUND, getBuilder(getPublicAPI(3).path("catalog").path(dataset.getId().getId()).path("collaboration").path("wiki")).buildGet()); CollaborationHelper collaborationHelper = l(CollaborationHelper.class); Wiki newWiki = new Wiki("sample wiki text", null); collaborationHelper.setWiki(dataset.getId().getId(), newWiki); // tags exist now Wiki wiki = expectSuccess(getBuilder(getPublicAPI(3).path("catalog").path(dataset.getId().getId()).path("collaboration").path("wiki")).buildGet(), Wiki.class); assertEquals(wiki.getText(), newWiki.getText()); // cleanup space newNamespaceService().deleteSpace(spacePath, NamespaceUtils.getVersion(spacePath, newNamespaceService())); }
private void createSpaceAndVDS(NamespaceKey spacePath, List<String> vdsPath) throws NamespaceException { // create space SpaceConfig spaceConfig = new SpaceConfig(); spaceConfig.setName(spacePath.getRoot()); newNamespaceService().addOrUpdateSpace(spacePath, spaceConfig); // create vds VirtualDataset virtualDataset = new VirtualDataset(); virtualDataset.setSql("select * from sys.version"); DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(vdsPath.get(vdsPath.size() - 1)); datasetConfig.setFullPathList(vdsPath); datasetConfig.setType(DatasetType.VIRTUAL_DATASET); datasetConfig.setVirtualDataset(virtualDataset); getSabotContext().getViewCreator(SystemUser.SYSTEM_USERNAME).createView(vdsPath, "select * from sys.version", null); } }