public JobResultsStore(final FileSystemPlugin plugin, final IndexedStore<JobId, JobResult> store, final BufferAllocator allocator) throws IOException { this.storageName = plugin.getName(); this.dfs = plugin.getFS(ImpersonationUtil.getProcessUserName()); this.jobStoreLocation = plugin.getConfig().getPath(); this.dfs.mkdirs(jobStoreLocation); this.store = store; this.allocator = allocator; this.jobResults = CacheBuilder.newBuilder() .maximumSize(100) .expireAfterAccess(15, TimeUnit.MINUTES) .build( new CacheLoader<JobId, JobData>() { @Override public JobData load(JobId key) throws Exception { final JobDataImpl jobDataImpl = new JobDataImpl(new LateJobLoader(key), key); return newJobDataReference(jobDataImpl); } }); }
@Override public boolean containerExists(NamespaceKey key) { final List<String> folderPath = key.getPathComponents(); try { return getFS(SYSTEM_USERNAME).isDirectory(PathUtils.toFSPath(resolveTableNameToValidPath(folderPath))); } catch (IOException e) { logger.debug("Failure reading path.", e); return false; } }
@Override public boolean datasetExists(NamespaceKey key) { final List<String> filePath = key.getPathComponents(); try { return getFS(SYSTEM_USERNAME).exists(PathUtils.toFSPath(resolveTableNameToValidPath(filePath))); } catch (IOException e) { logger.debug("Failure reading path.", e); return false; } }
protected boolean fileExists(String username, List<String> filePath) throws IOException { return getFS(username).isFile(PathUtils.toFSPath(resolveTableNameToValidPath(filePath))); }
@Override public boolean hasAccessPermission(String user, NamespaceKey key, DatasetConfig datasetConfig) { if (config.isImpersonationEnabled()) { if (datasetConfig.getReadDefinition() != null) { // allow accessing partial datasets final FileSystemWrapper userFs = getFS(user); final List<TimedRunnable<Boolean>> permissionCheckTasks = Lists.newArrayList(); permissionCheckTasks.addAll(getUpdateKeyPermissionTasks(datasetConfig, userFs)); permissionCheckTasks.addAll(getSplitPermissiomTasks(datasetConfig, userFs, user)); try { Stopwatch stopwatch = Stopwatch.createStarted(); final List<Boolean> accessPermissions = TimedRunnable.run("check access permission for " + key, logger, permissionCheckTasks, 16); stopwatch.stop(); logger.debug("Checking access permission for {} took {} ms", key, stopwatch.elapsed(TimeUnit.MILLISECONDS)); for (Boolean permission : accessPermissions) { if (!permission) { return false; } } } catch (IOException ioe) { throw UserException.dataReadError(ioe).build(logger); } } } return true; }
this.fs = getFS(SYSTEM_USERNAME);
public List<SchemaEntity> list(List<String> folderPath, String userName) { try { final List<FileStatus> files = Lists.newArrayList(getFS(userName).listStatus(PathUtils.toFSPath(resolveTableNameToValidPath(folderPath)))); final Set<List<String>> tableNames = Sets.newHashSet(); final NamespaceService ns = context.getNamespaceService(userName);
@Override public SourceState getState() { final FileSystemWrapper fs = getFS(ImpersonationUtil.getProcessUserName()); if (!fs.isPdfs()) { try { fs.listStatus(config.getPath()); return SourceState.GOOD; } catch (Exception e) { return SourceState.badState(e); } } else { return SourceState.GOOD; } }
try { try { files = DotFileUtil.getDotFiles(getFS(schemaConfig.getUserName()), config.getPath(), tableSchemaPath.get(tableSchemaPath.size() - 1), DotFileType.VIEW); } catch (AccessControlException e) { if (!schemaConfig.getIgnoreAuthErrors()) {
public Iterable<String> getSubPartitions(List<String> table, List<String> partitionColumns, List<String> partitionValues, SchemaConfig schemaConfig ) throws PartitionNotFoundException { List<FileStatus> fileStatuses; try { Path fullPath = PathUtils.toFSPath(resolveTableNameToValidPath(table)); fileStatuses = getFS(schemaConfig.getUserName()).list(fullPath, false); } catch (IOException e) { throw new PartitionNotFoundException("Error finding partitions for table " + table, e); } return new SubDirectoryList(fileStatuses); }
/** * Fetches a single item from the filesystem plugin */ public SchemaEntity get(List<String> path, String userName) { try { final FileStatus status = getFS(userName).getFileStatus(PathUtils.toFSPath(resolveTableNameToValidPath(path))); final Set<List<String>> tableNames = Sets.newHashSet(); final NamespaceService ns = context.getNamespaceService(userName); final NamespaceKey folderNSKey = new NamespaceKey(path); if (ns.exists(folderNSKey)) { for(NameSpaceContainer entity : ns.list(folderNSKey)) { if (entity.getType() == Type.DATASET) { tableNames.add(resolveTableNameToValidPath(entity.getDataset().getFullPathList())); } } } List<String> p = PathUtils.toPathComponents(status.getPath()); return getSchemaEntity(status, tableNames, p); } catch (IOException | NamespaceException e) { throw new RuntimeException(e); } }
@Override public void dropView(SchemaConfig schemaConfig, List<String> tableSchemaPath) throws IOException { if(!getMutability().hasMutationCapability(MutationType.VIEW, schemaConfig.isSystemUser())) { throw UserException.parseError() .message("Unable to drop view. Schema [%s] is immutable for this user.", this.name) .build(logger); } getFS(schemaConfig.getUserName()).delete(getViewPath(tableSchemaPath), false); }
@Override public boolean createOrUpdateView(NamespaceKey key, View view, SchemaConfig schemaConfig) throws IOException { if(!getMutability().hasMutationCapability(MutationType.VIEW, schemaConfig.isSystemUser())) { throw UserException.parseError() .message("Unable to create view. Schema [%s] is immutable for this user.", key.getParent()) .build(logger); } Path viewPath = getViewPath(key.getPathComponents()); FileSystemWrapper fs = getFS(schemaConfig.getUserName()); boolean replaced = fs.exists(viewPath); final FsPermission viewPerms = new FsPermission(schemaConfig.getOption(ExecConstants.NEW_VIEW_DEFAULT_PERMS_KEY).getStringVal()); try (OutputStream stream = FileSystemWrapper.create(fs, viewPath, viewPerms)) { lpPersistance.getMapper().writeValue(stream, view); } return replaced; }
FileSystemWrapper fs = getFS((user != null) ? user : userName); FileSelection fileSelection = FileSelection.create(fs, fullPath); String tableName = datasetPath.getName();
public JobDataFragment previewData(FileFormat format, NamespacePath namespacePath, boolean useFormatLocation) { final NamespaceKey key = namespacePath.toNamespaceKey(); final FileSystemPlugin plugin = getPlugin(key); final FileSystemWrapper fs = plugin.getFS(securityContext.getUserPrincipal().getName()); final Path path = FileSelection.getPathBasedOnFullPath(plugin.resolveTableNameToValidPath(key.getPathComponents()));
FileSystemWrapper fs = getFS(schemaConfig.getUserName()); List<String> fullPath = resolveTableNameToValidPath(tableSchemaPath); FileSelection fileSelection;
private FileFormat detectFileFormat(NamespaceKey key) { final FileSystemPlugin plugin = getPlugin(key); final FileSystemWrapper fs = plugin.getFS(securityContext.getUserPrincipal().getName()); final Path path = FileSelection.getPathBasedOnFullPath(plugin.resolveTableNameToValidPath(key.getPathComponents()));
@Override public VectorAccessible setup(VectorAccessible accessible) throws Exception { final BatchSchema schema = accessible.getSchema(); if(!schema.equals(RecordWriter.SCHEMA)){ throw new IllegalStateException(String.format("Incoming record writer schema doesn't match intended. Expected %s, received %s", RecordWriter.SCHEMA, schema)); } fs = config.getPlugin().getFS(config.getUserName()); // replacement expression. LogicalExpression replacement; if (config.getTempLocation() != null) { replacement = new FunctionCall("REGEXP_REPLACE", ImmutableList.<LogicalExpression>of( SchemaPath.getSimplePath(RecordWriter.PATH.getName()), new ValueExpressions.QuotedString(Pattern.quote(config.getTempLocation())), new ValueExpressions.QuotedString(config.getFinalLocation()) )); } else { replacement = SchemaPath.getSimplePath(RecordWriter.PATH.getName()); } Project projectConfig = new Project(ImmutableList.of( new NamedExpression(SchemaPath.getSimplePath(RecordWriter.FRAGMENT.getName()), new FieldReference(RecordWriter.FRAGMENT.getName())), new NamedExpression(SchemaPath.getSimplePath(RecordWriter.RECORDS.getName()), new FieldReference(RecordWriter.RECORDS.getName())), new NamedExpression(replacement, new FieldReference(RecordWriter.PATH.getName())), new NamedExpression(SchemaPath.getSimplePath(RecordWriter.METADATA.getName()), new FieldReference(RecordWriter.METADATA.getName())), new NamedExpression(SchemaPath.getSimplePath(RecordWriter.PARTITION.getName()), new FieldReference(RecordWriter.PARTITION.getName())), new NamedExpression(SchemaPath.getSimplePath(RecordWriter.FILESIZE.getName()), new FieldReference(RecordWriter.FILESIZE.getName())) ), null); this.project = new ProjectOperator(context, projectConfig); return project.setup(accessible); }