/** * Tests whether the path is a directory. * * @param status the {@link URIStatus} status as the input of the command * @return true if the path is a directory or false if it is not a directory */ private boolean isDir(URIStatus status) { return status.isFolder(); }
/** * Tests whether the path is a file. * * @param status the {@link URIStatus} status as the input of the command * @return true if the path is a file or false if it is not a file */ private boolean isFile(URIStatus status) { return !status.isFolder(); }
/** * Tests whether the directory is not empty. * * @param status the {@link URIStatus} status as the input of the command * @return true if the directory is not empty or false if it is empty */ private boolean isNonEmptyDir(URIStatus status) { return status.isFolder() && status.getLength() > 0; }
/** * Tests whether the file is zero length. * * @param status the {@link URIStatus} status as the input of the command * @return true if the file is zero length or false if it is not zero length */ private boolean isZeroLengthFile(URIStatus status) { return !status.isFolder() && status.getLength() == 0; }
/** * @param source an Alluxio URI * @param fileSystem the Alluxio file system * @return whether the URI is a file or a directory which contains files (including recursively) * @throws Exception if an unexpected exception occurs */ private static boolean hasFiles(AlluxioURI source, FileSystem fileSystem) throws Exception { Stack<AlluxioURI> dirsToCheck = new Stack<>(); dirsToCheck.add(source); while (!dirsToCheck.isEmpty()) { try { for (URIStatus status : fileSystem.listStatus(dirsToCheck.pop())) { if (!status.isFolder()) { return true; } dirsToCheck.push(new AlluxioURI(status.getPath())); } } catch (FileDoesNotExistException e) { // This probably means another worker has deleted the directory already, so we can probably // return false here. To be safe though, we will fall through and complete the search. } } return false; }
@Override protected void runPlainPath(AlluxioURI plainPath, CommandLine cl) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(plainPath); if (status.isFolder()) { throw new FileDoesNotExistException( ExceptionMessage.PATH_MUST_BE_FILE.getMessage(plainPath.getPath())); } String str = calculateChecksum(plainPath); System.out.println("md5sum: " + str + "\n"); }
private void queueNonPersistedRecursive(URIStatus status, Queue<AlluxioURI> toPersist) throws AlluxioException, IOException { AlluxioURI uri = new AlluxioURI(status.getPath()); if (status.isFolder()) { List<URIStatus> statuses = mFileSystem.listStatus(uri); for (URIStatus s : statuses) { queueNonPersistedRecursive(s, toPersist); } } else if (!status.isPersisted()) { toPersist.add(uri); } }
private void checkBucketIsAlluxioDirectory(String bucketPath) throws S3Exception { try { URIStatus status = mFileSystem.getStatus(new AlluxioURI(bucketPath)); if (!status.isFolder()) { throw new InvalidPathException("Bucket name is not a valid Alluxio directory."); } } catch (Exception e) { throw toBucketS3Exception(e, bucketPath); } }
/** * Creates a directory in the Alluxio filesystem space. It will not throw any exception if the * destination directory already exists. * * @param dstPath the {@link AlluxioURI} of the destination directory which will be created */ private void createDstDir(AlluxioURI dstPath) throws AlluxioException, IOException { try { mFileSystem.createDirectory(dstPath); } catch (FileAlreadyExistsException e) { // it's fine if the directory already exists } URIStatus dstStatus = mFileSystem.getStatus(dstPath); if (!dstStatus.isFolder()) { throw new InvalidPathException(ExceptionMessage.DESTINATION_CANNOT_BE_FILE.getMessage()); } }
@Override public FileStatus[] listStatus(Path path) throws IOException { LOG.debug("listStatus({})", path); if (mStatistics != null) { mStatistics.incrementReadOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); List<URIStatus> statuses; try { statuses = mFileSystem.listStatus(uri); } catch (FileDoesNotExistException e) { throw new FileNotFoundException(HadoopUtils.getPathWithoutScheme(path)); } catch (AlluxioException e) { throw new IOException(e); } FileStatus[] ret = new FileStatus[statuses.size()]; for (int k = 0; k < statuses.size(); k++) { URIStatus status = statuses.get(k); ret[k] = new FileStatus(status.getLength(), status.isFolder(), getReplica(status), status.getBlockSizeBytes(), status.getLastModificationTimeMs(), status.getCreationTimeMs(), new FsPermission((short) status.getMode()), status.getOwner(), status.getGroup(), new Path(mAlluxioHeader + status.getPath())); } return ret; }
@Override protected void runPlainPath(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException { // TODO(calvin): Remove explicit state checking. boolean recursive = cl.hasOption("R"); if (!mFileSystem.exists(path)) { throw new FileDoesNotExistException(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(path)); } if (!recursive && mFileSystem.getStatus(path).isFolder()) { throw new IOException( path.getPath() + " is a directory, to remove it, please use \"rm -R <path>\""); } boolean isAlluxioOnly = cl.hasOption(REMOVE_ALLUXIO_ONLY.getLongOpt()); DeletePOptions options = DeletePOptions.newBuilder().setRecursive(recursive).setAlluxioOnly(isAlluxioOnly) .setUnchecked(cl.hasOption(REMOVE_UNCHECKED_OPTION_CHAR)).build(); mFileSystem.delete(path, options); if (!isAlluxioOnly) { System.out.println(path + " has been removed"); } else { System.out.println(path + " has been removed from Alluxio space"); } }
@Override protected void runPlainPath(AlluxioURI plainPath, CommandLine cl) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(plainPath); if (status.isFolder()) { throw new IOException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(plainPath)); } OpenFilePOptions options = OpenFilePOptions.getDefaultInstance(); try (FileInStream is = mFileSystem.openFile(plainPath, options)) { long bytesToRead; if (status.getLength() > mNumOfBytes) { bytesToRead = mNumOfBytes; } else { bytesToRead = status.getLength(); } byte[] buf = new byte[(int) bytesToRead]; int read = is.read(buf); if (read != -1) { System.out.write(buf, 0, read); } } }
@Override protected void runPlainPath(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(path); if (status.isFolder()) { throw new FileDoesNotExistException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(path)); } OpenFilePOptions options = OpenFilePOptions.getDefaultInstance(); byte[] buf = new byte[Constants.MB]; try (FileInStream is = mFileSystem.openFile(path, options)) { int read = is.read(buf); while (read != -1) { System.out.write(buf, 0, read); read = is.read(buf); } } }
@Override public boolean setReplication(Path path, short replication) throws IOException { AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); try { if (!mFileSystem.exists(uri) || mFileSystem.getStatus(uri).isFolder()) { return false; } mFileSystem.setAttribute(uri, (SetAttributePOptions) SetAttributePOptions.newBuilder() .setReplicationMin(replication).build()); return true; } catch (AlluxioException e) { throw new IOException(e); } }
private long[] countHelper(AlluxioURI path) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(path); if (!status.isFolder()) { return new long[]{ 1L, 0L, status.getLength() }; } long[] rtn = new long[]{ 0L, 1L, 0L }; List<URIStatus> statuses; try { statuses = mFileSystem.listStatus(path); } catch (AlluxioException e) { throw new IOException(e.getMessage()); } for (URIStatus uriStatus : statuses) { long[] toAdd = countHelper(new AlluxioURI(uriStatus.getPath())); rtn[0] += toAdd[0]; rtn[1] += toAdd[1]; rtn[2] += toAdd[2]; } return rtn; }
/** * Loads a file or directory in Alluxio space, makes it resident in memory. * * @param filePath The {@link AlluxioURI} path to load into Alluxio memory * @throws AlluxioException when Alluxio exception occurs * @throws IOException when non-Alluxio exception occurs */ private void load(AlluxioURI filePath, int replication) throws AlluxioException, IOException, InterruptedException { URIStatus status = mFileSystem.getStatus(filePath); if (status.isFolder()) { List<URIStatus> statuses = mFileSystem.listStatus(filePath); for (URIStatus uriStatus : statuses) { AlluxioURI newPath = new AlluxioURI(uriStatus.getPath()); load(newPath, replication); } } else { Thread thread = JobGrpcClientUtils.createProgressThread(System.out); thread.start(); try { JobGrpcClientUtils.run(new LoadConfig(filePath.getPath(), replication), 3, mFsContext.getConf()); } finally { thread.interrupt(); } } System.out.println(filePath + " loaded"); }
/** * Returns a list of files at a given {@code path}. * * @param fs a {@link FileSystem} handler * @param path a path in alluxio file system * @return a list of strings representing the file names under the given path */ public static List<String> listFiles(FileSystem fs, String path) { try { List<URIStatus> statuses = fs.listStatus(new AlluxioURI(path)); List<String> res = new ArrayList<>(); for (URIStatus status : statuses) { res.add(status.getPath()); if (status.isFolder()) { res.addAll(listFiles(fs, status.getPath())); } } return res; } catch (IOException | AlluxioException e) { throw new RuntimeException(e); } }
private void printLsString(URIStatus status, boolean hSize) { // detect the extended acls boolean hasExtended = status.getAcl().hasExtended() || !status.getDefaultAcl().isEmpty(); System.out.print(formatLsString(hSize, SecurityUtils.isSecurityEnabled(mFsContext.getConf()), status.isFolder(), FormatUtils.formatMode((short) status.getMode(), status.isFolder(), hasExtended), status.getOwner(), status.getGroup(), status.getLength(), status.getLastModificationTimeMs(), status.getInAlluxioPercentage(), status.getPersistenceState(), status.getPath(), mFsContext.getConf().get(PropertyKey.USER_DATE_FORMAT_PATTERN))); }
@Override public FileInStream openFile(AlluxioURI path, OpenFilePOptions options) throws FileDoesNotExistException, IOException, AlluxioException { checkUri(path); URIStatus status = getStatus(path); if (status.isFolder()) { throw new FileDoesNotExistException( ExceptionMessage.CANNOT_READ_DIRECTORY.getMessage(status.getName())); } InStreamOptions inStreamOptions = new InStreamOptions(status, options, mFsContext.getConf()); return new FileInStream(status, inStreamOptions, mFsContext); }
/** * Creates a new instance of {@link alluxio.util.webui.UIFileInfo}. * * @param status underlying {@link URIStatus} */ public UIFileInfo(URIStatus status) { // detect the extended acls boolean hasExtended = status.getAcl().hasExtended() || !status.getDefaultAcl().isEmpty(); mId = status.getFileId(); mName = status.getName(); mAbsolutePath = status.getPath(); mBlockSizeBytes = status.getBlockSizeBytes(); mSize = status.getLength(); mCreationTimeMs = status.getCreationTimeMs(); mLastModificationTimeMs = status.getLastModificationTimeMs(); mInAlluxio = (100 == status.getInAlluxioPercentage()); mInAlluxioPercentage = status.getInAlluxioPercentage(); mIsDirectory = status.isFolder(); mPinned = status.isPinned(); mOwner = status.getOwner(); mGroup = status.getGroup(); mMode = FormatUtils.formatMode((short) status.getMode(), status.isFolder(), hasExtended); mPersistenceState = status.getPersistenceState(); mFileLocations = new ArrayList<>(); }