@VisibleForTesting public Catalog getSystemUserCatalog() { return getCatalog(SchemaConfig.newBuilder(SystemUser.SYSTEM_USERNAME).build()); }
public SchemaConfig build() { return new SchemaConfig( new AuthorizationContext(username, ignoreAuthErrors), defaultSchema, optionManager, viewExpansionContext, exposeInternalSources, validityChecker); } }
public MetadataRequestOptions cloneWith(String newUser, NamespaceKey newDefaultSchema) { SchemaConfig newSchemaConfig = SchemaConfig.newBuilder(newUser) .defaultSchema(newDefaultSchema) .exposeInternalSources(schemaConfig.exposeInternalSources()) .setIgnoreAuthErrors(schemaConfig.getIgnoreAuthErrors()) .optionManager(schemaConfig.getOptions()) .setViewExpansionContext(schemaConfig.getViewExpansionContext()) .setDatasetValidityChecker(schemaConfig.getDatasetValidityChecker()) .build(); return new MetadataRequestOptions(newSchemaConfig, statsCollector, maxRequestTime); } }
@Override public TranslatableTable apply(final List<Object> arguments) { try { SourceTableDefinition definition = plugin.getDatasetWithOptions(new NamespaceKey (tableSchemaPath), new TableInstance(sig, arguments), schemaConfig.getIgnoreAuthErrors(), schemaConfig.getUserName()); if(definition == null){ throw UserException.validationError().message("Unable to read table %s using provided options.", new NamespaceKey(tableSchemaPath).toString()).build(logger); } return new MaterializedDatasetTable(plugin, schemaConfig.getUserName(), new QuietAccessor(definition)); } catch (Exception e) { throw new RuntimeException(e); } }
@Override public boolean createOrUpdateView(NamespaceKey key, View view, SchemaConfig schemaConfig) throws IOException { if(!getMutability().hasMutationCapability(MutationType.VIEW, schemaConfig.isSystemUser())) { throw UserException.parseError() .message("Unable to create view. Schema [%s] is immutable for this user.", key.getParent()) .build(logger); } Path viewPath = getViewPath(key.getPathComponents()); FileSystemWrapper fs = getFS(schemaConfig.getUserName()); boolean replaced = fs.exists(viewPath); final FsPermission viewPerms = new FsPermission(schemaConfig.getOption(ExecConstants.NEW_VIEW_DEFAULT_PERMS_KEY).getStringVal()); try (OutputStream stream = FileSystemWrapper.create(fs, viewPath, viewPerms)) { lpPersistance.getMapper().writeValue(stream, view); } return replaced; }
@Override public NamespaceKey getDefaultSchema() { return options.getSchemaConfig().getDefaultSchema(); }
/** * Checks if the entry is valid. * * @param options metadata request options * @param config dataset config * @return true iff entry is valid */ boolean isStillValid(MetadataRequestOptions options, DatasetConfig config) { final NamespaceKey key = new NamespaceKey(config.getFullPathList()); final Long updateTime = localUpdateTime.getIfPresent(key); final long currentTime = System.currentTimeMillis(); final long expiryTime = msp.getMetadataPolicy().getDatasetDefinitionExpireAfterMs(); // check if the entry is expired if ( // request marks this expired options.getMaxRequestTime() < currentTime || // dataset was locally updated too long ago (or never) ((updateTime == null || updateTime + expiryTime < currentTime) && // AND dataset was globally updated too long ago lastRefreshTime + expiryTime < currentTime) || // request marks this dataset as invalid !options.getSchemaConfig().getDatasetValidityChecker().apply(config) ) { return false; } return true; }
try { try { files = DotFileUtil.getDotFiles(getFS(schemaConfig.getUserName()), config.getPath(), tableSchemaPath.get(tableSchemaPath.size() - 1), DotFileType.VIEW); } catch (AccessControlException e) { if (!schemaConfig.getIgnoreAuthErrors()) { logger.debug(e.getMessage()); throw UserException.permissionError(e) return new ViewTable(new NamespaceKey(tableSchemaPath), f.getView(lpPersistance), f.getOwner(), null); } catch (AccessControlException e) { if (!schemaConfig.getIgnoreAuthErrors()) { logger.debug(e.getMessage()); throw UserException.permissionError(e)
@Override public NamespaceKey resolveToDefault(NamespaceKey key) { if(options.getSchemaConfig().getDefaultSchema() == null) { return null; } return new NamespaceKey( ImmutableList.copyOf( Iterables.concat( options.getSchemaConfig().getDefaultSchema().getPathComponents(), key.getPathComponents()))); }
private Catalog createCatalog(String userName) { return catalogService.getCatalog(SchemaConfig.newBuilder(userName).build()); }
plugin.checkAccess(key, datasetConfig, options); final NamespaceKey canonicalKey = new NamespaceKey(datasetConfig.getFullPathList()); final NamespaceTable namespaceTable = new NamespaceTable(new TableMetadataImpl(plugin.getId(), datasetConfig, options.getSchemaConfig().getUserName(), DatasetSplitsPointer.of(userNamespaceService, datasetConfig))); options.getStatsCollector().addDatasetStat(canonicalKey.getSchemaPath(), MetadataAccessType.CACHED_METADATA.name(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); return namespaceTable; tableDefinition = plugin.getTable(key, datasetConfig, options.getSchemaConfig().getIgnoreAuthErrors()); } catch (Exception ex) { throw UserException.validationError(ex).message("Failure while retrieving dataset [%s].", key).build(logger); TableMetadata metadata = new TableMetadataImpl(plugin.getId(), newDatasetConfig, options.getSchemaConfig().getUserName(), MaterializedSplitsPointer.of(splits, splits.size())); return new NamespaceTable(metadata); } catch(UserException ex) {
@Override public Catalog resolveCatalog(String username) { return new CatalogImpl(context, options.cloneWith(username, options.getSchemaConfig().getDefaultSchema()), pluginRetriever, sourceModifier); }
private Catalog createCatalog() { return catalogService.getCatalog(SchemaConfig.newBuilder(security.getUserPrincipal().getName()).build()); }
public RelDataType getRowType(final NamespaceKey path) { DremioTable table = catalogService.getCatalog(SchemaConfig.newBuilder(SystemUser.SYSTEM_USERNAME).build()).getTable(path); Preconditions.checkNotNull(table, "Unknown dataset %s", path); return table.getRowType(JavaTypeFactoryImpl.INSTANCE); }
@Override @RequestScoped public Catalog provide() { return catalogService.getCatalog(SchemaConfig.newBuilder(context.getUserPrincipal().getName()).build()); }
public List<String> getColumnList(final String username, DatasetPath path) { Catalog catalog = catalogService.getCatalog(SchemaConfig.newBuilder(context.getUserPrincipal().getName()).build()); DremioTable table = catalog.getTable(path.toNamespaceKey()); return table.getRowType(SqlTypeFactoryImpl.INSTANCE).getFieldNames(); }
.getCatalog(SchemaConfig.newBuilder(SystemUser.SYSTEM_USERNAME).build()) .createDataset(new NamespaceKey(getMaterializationPath(materialization)), datasetMutator);
protected Catalog cat() { return l(CatalogService.class).getCatalog(SchemaConfig.newBuilder(SYSTEM_USERNAME).build()); }
private List<String> getPartitionColumnsForDataSet(String folder) throws Exception { URL stream = (TestParquetPartitionColumns.class.getClassLoader() .getResource(folder)); File fileDir = new File(stream.getFile()); DatasetPath parquet = new DatasetPath(ImmutableList.of("dfs", fileDir.getAbsolutePath())); DatasetConfig config = addDataSet(parquet); DremioTable table = p(CatalogService.class).get().getCatalog(SchemaConfig.newBuilder (DEFAULT_USERNAME) .build()) .getTable(config.getId().getId()); return table.getDatasetConfig().getReadDefinition() .getPartitionColumnsList(); }
/** * @throws UserException if reflection goal is invalid */ public void validate(ReflectionGoal goal) { ReflectionUtils.validateReflectionGoalWithoutSchema(goal); // The dataset that the reflection refers to must exist. DatasetConfig dataset = namespaceService.get().findDatasetByUUID(goal.getDatasetId()); Preconditions.checkNotNull(dataset, "datasetId must reference an existing dataset"); Catalog catalog = catalogService.get().getCatalog(SchemaConfig.newBuilder(SystemUser.SYSTEM_USERNAME).build()); DremioTable table = catalog.getTable(new NamespaceKey(dataset.getFullPathList())); List<ViewFieldType> schemaFields = ViewFieldsHelper.getBatchSchemaFields(table.getSchema()); Map<String, ViewFieldType> schemaMap = Maps.newHashMap(); for (ViewFieldType type : schemaFields) { schemaMap.put(type.getName(), type); } // Array, List, Map, and Union types cannot be used as dimension or measure field. They also cannot be used for sorting, partitioning or distribution in raw reflections. validateFields(goal.getDetails().getDisplayFieldList(), schemaMap, "Display", false); validateMeasures(goal.getDetails().getMeasureFieldList(), schemaMap, "Measure"); validateFields(goal.getDetails().getDistributionFieldList(), schemaMap, "Distribution", true); validateFields(goal.getDetails().getSortFieldList(), schemaMap, "Sort", true); validateFields(goal.getDetails().getPartitionFieldList(), schemaMap, "Partition", true); validateDimensions(goal.getDetails().getDimensionFieldList(), schemaMap); }