Refine search
List<FileStatus> result = new ArrayList<FileStatus>(); try { FileStatus fileStatus = fs.getFileStatus(path); FileUtils.listStatusRecursively(fs, fileStatus, filter, result); } catch (IOException e) { StringBuilder sb = new StringBuilder(path.toUri().getPath()); for (int i = 0; i < level; i++) { sb.append(Path.SEPARATOR).append("*"); Path pathPattern = new Path(path, sb.toString()); if (!allLevelsBelow) { return Lists.newArrayList(fs.globStatus(pathPattern, filter)); for (FileStatus status : fs.globStatus(pathPattern)) { if (filter.accept(status.getPath())) { results.add(status); if (status.isDirectory()) { queue.add(status); for (FileStatus child : fs.listStatus(status.getPath())) { if (filter.accept(child.getPath())) { results.add(child);
@Override public Map<FileStatus, String> findAllTopics(Path input, PathFilter filter, String topicSubdir, FileSystem fs) throws IOException { Map<FileStatus, String> topics = new HashMap<FileStatus, String>(); for (FileStatus f : fs.listStatus(input)) { if (f.isDir() && filter.accept(f.getPath())) { LOG.info("found topic: " + f.getPath().getName()); topics.put(fs.getFileStatus(f.getPath()), f.getPath().getName()); } } return topics; }
protected Set<Path> selectFiles(final FileSystem hdfs, final Path inputPath, Set<Path> filesVisited) throws IOException { if (null == filesVisited) { filesVisited = new HashSet<>(); } if (!hdfs.exists(inputPath)) { throw new IOException("Selection directory " + inputPath.toString() + " doesn't appear to exist!"); } final Set<Path> files = new HashSet<>(); FileStatus inputStatus = hdfs.getFileStatus(inputPath); if (inputStatus.isDirectory()) { for (final FileStatus file : hdfs.listStatus(inputPath)) { final Path canonicalFile = file.getPath(); if (!filesVisited.add(canonicalFile)) { // skip files we've already seen (may be looping directory links) continue; } if (!file.isDirectory() && processorConfig.getPathFilter(inputPath).accept(canonicalFile)) { files.add(canonicalFile); if (getLogger().isDebugEnabled()) { getLogger().debug(this + " selected file at path: " + canonicalFile.toString()); } } } } else if (inputStatus.isFile()) { files.add(inputPath); } return files; }
private static List<FileStatus> listMostNestedPathRecursivelyHelper(FileSystem fs, List<FileStatus> files, FileStatus fileStatus, PathFilter fileFilter) throws IOException { if (fileStatus.isDirectory()) { FileStatus[] curFileStatus = fs.listStatus(fileStatus.getPath()); if (ArrayUtils.isEmpty(curFileStatus)) { files.add(fileStatus); } else { for (FileStatus status : curFileStatus) { listMostNestedPathRecursivelyHelper(fs, files, status, fileFilter); } } } else if (fileFilter.accept(fileStatus.getPath())) { files.add(fileStatus); } return files; }
String pathPatternString = pathPattern.toUri().getPath(); List<String> flattenedPatterns = GlobExpander.expand(pathPatternString); Path absPattern = fixRelativePart(new Path( flatPattern.isEmpty() ? Path.CUR_DIR : flatPattern)); getPathComponents(absPattern.toUri().getPath()); rootPlaceholder = new FileStatus(0, true, 0, 0, 0, new Path(scheme, rootPlaceholder = new FileStatus(0, true, 0, 0, 0, new Path(scheme, authority, Path.SEPARATOR)); candidate.setPath(new Path(candidate.getPath(), component)); if (filter.accept(status.getPath())) { results.add(status);
/** * Will assert that actual files match expected. * @param expectedFiles - suffixes of expected Paths. Must be the same length * @param rootPath - table or partition root where to start looking for actual files, recursively */ void assertExpectedFileSet(Set<String> expectedFiles, String rootPath) throws Exception { int suffixLength = 0; for(String s : expectedFiles) { if(suffixLength > 0) { assert suffixLength == s.length() : "all entries must be the same length. current: " + s; } suffixLength = s.length(); } FileSystem fs = FileSystem.get(hiveConf); Set<String> actualFiles = new HashSet<>(); RemoteIterator<LocatedFileStatus> remoteIterator = fs.listFiles(new Path(rootPath), true); while (remoteIterator.hasNext()) { LocatedFileStatus lfs = remoteIterator.next(); if(!lfs.isDirectory() && org.apache.hadoop.hive.common.FileUtils.HIDDEN_FILES_PATH_FILTER.accept(lfs.getPath())) { String p = lfs.getPath().toString(); actualFiles.add(p.substring(p.length() - suffixLength, p.length())); } } Assert.assertEquals("Unexpected file list", expectedFiles, actualFiles); } void checkExpected(List<String> rs, String[][] expected, String msg, Logger LOG, boolean checkFileName) {
String lastRelDir = null; HashSet<Path> results = new HashSet<Path>(); String relRoot = Path.getPathWithoutSchemeAndAuthority(path).toString(); if (!relRoot.endsWith(Path.SEPARATOR)) { relRoot += Path.SEPARATOR; RemoteIterator<LocatedFileStatus> allFiles = fs.listFiles(path, true); while (allFiles.hasNext()) { LocatedFileStatus lfs = allFiles.next(); Path lfsPath = lfs.getPath(); Path dirPath = Path.getPathWithoutSchemeAndAuthority(lfsPath); String dir = dirPath.toString(); if (!dir.startsWith(relRoot)) { Path candidate = new Path(relRoot, subDir); if (!filter.accept(candidate)) { continue; results.add(fs.makeQualified(candidate));
/** * Filter files/directories in the given path using the user-supplied path * filter. Results are added to the given array <code>results</code>. * @throws FileNotFoundException when the path does not exist * @throws IOException see specific implementation */ private void listStatus(ArrayList<FileStatus> results, Path f, PathFilter filter) throws FileNotFoundException, IOException { FileStatus listing[] = listStatus(f); Preconditions.checkNotNull(listing, "listStatus should not return NULL"); for (int i = 0; i < listing.length; i++) { if (filter.accept(listing[i].getPath())) { results.add(listing[i]); } } }
private void appendFiles(FileSystem fs, Path path, Path base, PathFilter pathFilter, KeyProvider<K> keyProvider, Map<Path, K> addedFiles) throws IOException { if (fs.isFile(path)) { if (pathFilter == null || pathFilter.accept(path)) { final K key = keyProvider.getKey(fs, path, base); appendFile(key, fs, path); addedFiles.put(path, key); } } else { try { final FileStatus[] status = fs.listStatus(path); for (final FileStatus stat : status) { appendFiles(fs, stat.getPath(), base, pathFilter, keyProvider, addedFiles); } } catch (final Throwable e) { System.err.println("Failed listing status on path: " + path); } } }
List<FileStatus> familyDirs = FSUtils.listStatusWithStatusFilter(fs, dd, familyFilter); if (familyDirs == null) { if (!fs.exists(dd)) { LOG.warn("Skipping region because it no longer exists: " + dd); } else { errors.progress(); Path family = familyDir.getPath(); if (family.getName().equals(HConstants.RECOVERED_EDITS_DIR)) { continue; FileStatus[] familyStatus = fs.listStatus(family); for (FileStatus sfStatus : familyStatus) { if (null != errors) { errors.progress(); Path sf = sfStatus.getPath(); if (sfFilter == null || sfFilter.accept(sf)) { regionStoreFileMap.put( sf.getName(), sf);
FileStatus[] fileStatuses = getUserGroupInformation().doAs((PrivilegedExceptionAction<FileStatus[]>) () -> hdfs.listStatus(dir)); for (final FileStatus file : fileStatuses) { if (files.size() >= MAX_WORKING_QUEUE_SIZE) { final Path canonicalFile = file.getPath(); if (file.isDirectory() && processorConfig.getRecurseSubdirs()) { files.addAll(selectFiles(hdfs, canonicalFile, filesVisited)); } else if (!file.isDirectory() && processorConfig.getPathFilter(dir).accept(canonicalFile)) { final long fileAge = System.currentTimeMillis() - file.getModificationTime(); if (processorConfig.getMinimumAge() < fileAge && fileAge < processorConfig.getMaximumAge()) { getLogger().debug(this + " selected file at path: " + canonicalFile.toString());
@Override public FileStatus[] listStatus(Path path) throws IOException { try { List<String> fileNames = this.fsHelper.ls(HadoopUtils.toUriPath(path)); List<FileStatus> status = Lists.newArrayListWithCapacity(fileNames.size()); for (String name : fileNames) { Path filePath = new Path(name); if (VALID_PATH_FILTER.accept(filePath)) { status.add(getFileStatus(new Path(path, filePath))); } } return status.toArray(new FileStatus[status.size()]); } catch (FileBasedHelperException e) { throw new IOException(e); } }
private void listStatus(ArrayList<FileStatus> results, Path f, PathFilter filter) throws AccessControlException, FileNotFoundException, IOException { FileStatus[] listing = listStatus(f); if (listing != null) { for (int i = 0; i < listing.length; i++) { if (filter.accept(listing[i].getPath())) { results.add(listing[i]); } } } }
@Override public CarbonFile[] locationAwareListFiles(PathFilter pathFilter) throws IOException { if (null != fileStatus && fileStatus.isDirectory()) { List<FileStatus> listStatus = new ArrayList<>(); Path path = fileStatus.getPath(); RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path); while (iter.hasNext()) { LocatedFileStatus fileStatus = iter.next(); if (pathFilter.accept(fileStatus.getPath()) && fileStatus.getLen() > 0) { listStatus.add(fileStatus); } } return getFiles(listStatus.toArray(new FileStatus[listStatus.size()])); } return new CarbonFile[0]; }
private static ArrayList<FileStatus> traverseImpl(Storage storage, Path path, PathFilter filter) throws IOException { if (!storage.exists(path.toString())) { return new ArrayList<>(); } ArrayList<FileStatus> result = new ArrayList<>(); FileStatus[] statuses = storage.listStatus(path.toString()); for (FileStatus status : statuses) { if (status.isDirectory()) { result.addAll(traverseImpl(storage, status.getPath(), filter)); } else { if (filter.accept(status.getPath())) { result.add(status); } } } return result; }
public static List<FileStatus> listLocatedStatus(final FileSystem fs, final Path path, final PathFilter filter ) throws IOException { RemoteIterator<LocatedFileStatus> itr = fs.listLocatedStatus(path); List<FileStatus> result = new ArrayList<FileStatus>(); while(itr.hasNext()) { FileStatus stat = itr.next(); if (filter == null || filter.accept(stat.getPath())) { result.add(stat); } } return result; }
/** * A log file has a creation timestamp (in ms) in its file name ({@link #filenum}. This helper * method returns the creation timestamp from a given log file. It extracts the timestamp assuming * the filename is created with the {@link #computeFilename(long filenum)} method. * @return timestamp, as in the log file name. */ protected long getFileNumFromFileName(Path fileName) { checkNotNull(fileName, "file name can't be null"); if (!ourFiles.accept(fileName)) { throw new IllegalArgumentException( "The log file " + fileName + " doesn't belong to this WAL. (" + toString() + ")"); } final String fileNameString = fileName.toString(); String chompedPath = fileNameString.substring(prefixPathStr.length(), (fileNameString.length() - walFileSuffix.length())); return Long.parseLong(chompedPath); }
@Override public List<HdfsFileStatusWithId> listLocatedHdfsStatus( FileSystem fs, Path p, PathFilter filter) throws IOException { DistributedFileSystem dfs = ensureDfs(fs); DFSClient dfsc = dfs.getClient(); final String src = p.toUri().getPath(); DirectoryListing current = dfsc.listPaths(src, org.apache.hadoop.hdfs.protocol.HdfsFileStatus.EMPTY_NAME, true); if (current == null) { // the directory does not exist throw new FileNotFoundException("File " + p + " does not exist."); } final URI fsUri = fs.getUri(); List<HdfsFileStatusWithId> result = new ArrayList<HdfsFileStatusWithId>( current.getPartialListing().length); while (current != null) { org.apache.hadoop.hdfs.protocol.HdfsFileStatus[] hfss = current.getPartialListing(); for (int i = 0; i < hfss.length; ++i) { HdfsLocatedFileStatus next = (HdfsLocatedFileStatus)(hfss[i]); if (filter != null) { Path filterPath = next.getFullPath(p).makeQualified(fsUri, null); if (!filter.accept(filterPath)) continue; } LocatedFileStatus lfs = next.makeQualifiedLocated(fsUri, p); result.add(new HdfsFileStatusWithIdImpl(lfs, next.getFileId())); } current = current.hasMore() ? dfsc.listPaths(src, current.getLastName(), true) : null; } return result; }
public static List<String> getFiles(FileSystem fs, Path rootDir, List<String> files, PathFilter filter) throws IOException { RemoteIterator<LocatedFileStatus> it = fs.listFiles(rootDir, true); while (it.hasNext()) { LocatedFileStatus lfs = it.next(); if (lfs.isDirectory()) { continue; } // apply filter if (filter.accept(lfs.getPath())) { files.add(lfs.getPath().toString()); } } return files; }