static FileStatus[] listStatus(FileSystem fs, Path path) throws IOException { if (!fs.exists(path)) { return null; } return fs.listStatus(path); }
public static boolean isEmptyPath(Configuration job, Path dirPath) throws IOException { FileSystem inpFs = dirPath.getFileSystem(job); try { FileStatus[] fStats = inpFs.listStatus(dirPath, FileUtils.HIDDEN_FILES_PATH_FILTER); if (fStats.length > 0) { return false; } } catch(FileNotFoundException fnf) { return true; } return true; }
FileIterator(String locString) { try { if (locString != null) { Path loc = new Path(locString); fs = loc.getFileSystem(conf); files = fs.listStatus(loc, VALID_FILES_FILTER); } } catch (IOException e) { throw new RuntimeException(e); } }
protected void moveToOutputDirectory(FileSystem fs) throws Exception { LOGGER.info("Moving Segment Tar files from {} to: {}", _stagingDir + "/output/segmentTar", _outputDir); FileStatus[] segmentArr = fs.listStatus(new Path(_stagingDir + "/output/segmentTar")); for (FileStatus segment : segmentArr) { fs.rename(segment.getPath(), new Path(_outputDir, segment.getPath().getName())); } }
public void listDictSlicePath(FileSystem fs, FileStatus path, List<Path> list) throws IOException { if (path.isDirectory()) { for (FileStatus status : fs.listStatus(path.getPath())) { listDictSlicePath(fs, status, list); } } else { if (path.getPath().getName().startsWith(GlobalDictHDFSStore.IndexFormatV1.SLICE_PREFIX)) { list.add(path.getPath()); } } }
public static Collection<String> getAllFilesInPath(final Path path, final Configuration conf) throws IOException { List<String> filesInPath = new ArrayList<>(); FileSystem fs = path.getFileSystem(conf); FileStatus fileStatus = fs.getFileStatus(path); if (fileStatus.isDir()) { FileStatus[] fileStatuses = fs.listStatus(path, FileDump.HIDDEN_AND_SIDE_FILE_FILTER); for (FileStatus fileInPath : fileStatuses) { if (fileInPath.isDir()) { filesInPath.addAll(getAllFilesInPath(fileInPath.getPath(), conf)); } else { filesInPath.add(fileInPath.getPath().toString()); } } } else { filesInPath.add(path.toString()); } return filesInPath; } }
public static void listStatusRecursively(FileSystem fs, FileStatus fileStatus, PathFilter filter, List<FileStatus> results) throws IOException { if (fileStatus.isDir()) { for (FileStatus stat : fs.listStatus(fileStatus.getPath(), filter)) { listStatusRecursively(fs, stat, results); } } else { results.add(fileStatus); } }
/** * @param fs * @param rootdir * @return All the table directories under <code>rootdir</code>. Ignore non table hbase folders such as * .logs, .oldlogs, .corrupt folders. * @throws IOException */ public static List<Path> getLocalTableDirs(final FileSystem fs, final Path rootdir) throws IOException { // presumes any directory under hbase.rootdir is a table FileStatus[] dirs = fs.listStatus(rootdir, new UserTableDirFilter(fs)); List<Path> tabledirs = new ArrayList<>(dirs.length); for (FileStatus dir: dirs) { tabledirs.add(dir.getPath()); } return tabledirs; }
private void addJars(Configuration conf) throws IOException { if (!state.contains(MRCompactor.COMPACTION_JARS)) { return; } Path jarFileDir = new Path(state.getProp(MRCompactor.COMPACTION_JARS)); for (FileStatus status : this.fs.listStatus(jarFileDir)) { DistributedCache.addFileToClassPath(status.getPath(), conf, this.fs); } }
private TreeSet<String> getFilePath(Path p, String resPathPrefix) throws IOException { TreeSet<String> fileList = new TreeSet<>(); for (FileStatus fileStat : fs.listStatus(p)) { fileList.add(resPathPrefix + fileStat.getPath().getName()); } return fileList; }
public static Path getFilterOnlyPath(FileSystem fs, Path baseDir, final String filter) throws IOException { if (fs.exists(baseDir) == false) { return null; } FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() { @Override public boolean accept(Path path) { return path.getName().startsWith(filter); } }); if (fileStatus.length == 1) { return fileStatus[0].getPath(); } else { return null; } }
private long sizeOfPath(FileSystem fs, Path path) throws IOException { long size = 0; FileStatus[] statuses = fs.listStatus(path); if(statuses != null) { for(FileStatus status: statuses) { if(status.isDir()) size += sizeOfPath(fs, status.getPath()); else size += status.getLen(); } } return size; }
private boolean containMetaWals(ServerName serverName) throws IOException { Path logDir = new Path(master.getWALRootDir(), AbstractFSWALProvider.getWALDirectoryName(serverName.toString())); Path splitDir = logDir.suffix(AbstractFSWALProvider.SPLITTING_EXT); Path checkDir = master.getFileSystem().exists(splitDir) ? splitDir : logDir; return master.getFileSystem().listStatus(checkDir, META_FILTER).length > 0; }
private URI mostRecentInDir(final Path dir, final Pattern pattern) throws IOException { final PathFilter filter = new PathFilter() { @Override public boolean accept(Path path) { return pattern == null || pattern.matcher(path.getName()).matches(); } }; long modifiedTime = Long.MIN_VALUE; URI mostRecentURI = null; final FileSystem fs = dir.getFileSystem(config); for (FileStatus status : fs.listStatus(dir, filter)) { if (status.isFile()) { final long thisModifiedTime = status.getModificationTime(); if (thisModifiedTime >= modifiedTime) { modifiedTime = thisModifiedTime; mostRecentURI = status.getPath().toUri(); } } } return mostRecentURI; }
long getModTime(final HRegion r) throws IOException { FileStatus[] statuses = r.getRegionFileSystem().getFileSystem().listStatus( new Path(r.getRegionFileSystem().getRegionDir(), HRegionFileSystem.REGION_INFO_FILE)); assertTrue(statuses != null && statuses.length == 1); return statuses[0].getModificationTime(); }
private void checkRestoreTmpDir(Configuration conf, String restoreTmpDir, int expectedCount) throws IOException { FileSystem fs = FileSystem.get(conf); FileStatus[] subDirectories = fs.listStatus(new Path(restoreTmpDir)); assertNotNull(subDirectories); assertEquals(subDirectories.length, expectedCount); for (int i = 0; i < expectedCount; i++) { assertTrue(subDirectories[i].isDirectory()); } }
private FileStatus[] listFilesInDir(Path path) throws IOException { return dataFileSystem.listStatus(path, p -> { String name = p.getName(); return !name.startsWith("_") && !name.startsWith("."); }); }
private static List<Path> getTargetPathsFromSymlink(FileSystem fileSystem, Path symlinkDir) { try { FileStatus[] symlinks = fileSystem.listStatus(symlinkDir, HIDDEN_FILES_PATH_FILTER); List<Path> targets = new ArrayList<>(); for (FileStatus symlink : symlinks) { try (BufferedReader reader = new BufferedReader(new InputStreamReader(fileSystem.open(symlink.getPath()), StandardCharsets.UTF_8))) { CharStreams.readLines(reader).stream() .map(Path::new) .forEach(targets::add); } } return targets; } catch (IOException e) { throw new PrestoException(HIVE_BAD_DATA, "Error parsing symlinks from: " + symlinkDir, e); } }
@Override public List<String> getTableNames(String storeName, Predicate<String> predicate) throws IOException { List<String> names = Lists.newArrayList(); Path storePath = new Path(this.storeRootDir, storeName); if (!this.fs.exists(storePath)) { return names; } for (FileStatus status : this.fs.listStatus(storePath)) { if (predicate.apply(status.getPath().getName())) { names.add(status.getPath().getName()); } } return names; }
private static void getAllNestedAvroFiles(FileStatus dir, List<FileStatus> files, FileSystem fs) throws IOException { if (dir.isDirectory()) { FileStatus[] filesInDir = fs.listStatus(dir.getPath()); if (filesInDir != null) { for (FileStatus f : filesInDir) { getAllNestedAvroFiles(f, files, fs); } } } else if (dir.getPath().getName().endsWith(AVRO_SUFFIX)) { files.add(dir); } }