@Override public List<HdfsFileStatusWithId> listLocatedHdfsStatus( FileSystem fs, Path p, PathFilter filter) throws IOException { DistributedFileSystem dfs = ensureDfs(fs); DFSClient dfsc = dfs.getClient(); final String src = p.toUri().getPath(); DirectoryListing current = dfsc.listPaths(src, org.apache.hadoop.hdfs.protocol.HdfsFileStatus.EMPTY_NAME, true); if (current == null) { // the directory does not exist throw new FileNotFoundException("File " + p + " does not exist."); } final URI fsUri = fs.getUri(); List<HdfsFileStatusWithId> result = new ArrayList<HdfsFileStatusWithId>( current.getPartialListing().length); while (current != null) { org.apache.hadoop.hdfs.protocol.HdfsFileStatus[] hfss = current.getPartialListing(); for (int i = 0; i < hfss.length; ++i) { HdfsLocatedFileStatus next = (HdfsLocatedFileStatus)(hfss[i]); if (filter != null) { Path filterPath = next.getFullPath(p).makeQualified(fsUri, null); if (!filter.accept(filterPath)) continue; } LocatedFileStatus lfs = next.makeQualifiedLocated(fsUri, p); result.add(new HdfsFileStatusWithIdImpl(lfs, next.getFileId())); } current = current.hasMore() ? dfsc.listPaths(src, current.getLastName(), true) : null; } return result; }
h.put(p); DirectoryListing dl = dfs.getClient().listPaths(rootDir, HdfsFileStatus.EMPTY_NAME); HdfsFileStatus[] hfs = dl.getPartialListing();
/** * @return whether there is still remaing migration work for the next * round */ private void processPath(String fullPath, Result result) { for (byte[] lastReturnedName = HdfsFileStatus.EMPTY_NAME;;) { final DirectoryListing children; try { children = dfs.listPaths(fullPath, lastReturnedName, true); } catch(IOException e) { LOG.warn("Failed to list directory " + fullPath + ". Ignore the directory and continue.", e); return; } if (children == null) { return; } for (HdfsFileStatus child : children.getPartialListing()) { processRecursively(fullPath, child, result); } if (children.hasMore()) { lastReturnedName = children.getLastName(); } else { return; } } }
final DirectoryListing children; try { children = dfs.getClient().listPaths(childPath, lastReturnedName, false); } catch (IOException e) {
/** * Get a partial listing of the indicated directory * No block locations need to be fetched */ public DirectoryListing listPaths(String src, byte[] startAfter) throws IOException { return listPaths(src, startAfter, false); }
/** * Get a partial listing of the indicated directory * No block locations need to be fetched */ public DirectoryListing listPaths(String src, byte[] startAfter) throws IOException { return listPaths(src, startAfter, false); }
/** * Get a partial listing of the indicated directory * No block locations need to be fetched */ public DirectoryListing listPaths(String src, byte[] startAfter) throws IOException { return listPaths(src, startAfter, false); }
private DirListingIterator(Path p, boolean needLocation) throws IOException { this.src = Hdfs.this.getUriPath(p); this.needLocation = needLocation; // fetch the first batch of entries in the directory thisListing = dfs.listPaths( src, HdfsFileStatus.EMPTY_NAME, needLocation); if (thisListing == null) { // the directory does not exist throw new FileNotFoundException("File " + src + " does not exist."); } }
private DirListingIterator(Path p, boolean needLocation) throws IOException { this.src = Hdfs.this.getUriPath(p); this.needLocation = needLocation; // fetch the first batch of entries in the directory thisListing = dfs.listPaths( src, HdfsFileStatus.EMPTY_NAME, needLocation); if (thisListing == null) { // the directory does not exist throw new FileNotFoundException("File " + src + " does not exist."); } }
private DirListingIterator(Path p, boolean needLocation) throws IOException { this.src = Hdfs.this.getUriPath(p); this.needLocation = needLocation; // fetch the first batch of entries in the directory thisListing = dfs.listPaths( src, HdfsFileStatus.EMPTY_NAME, needLocation); if (thisListing == null) { // the directory does not exist throw new FileNotFoundException("File " + src + " does not exist."); } }
private DirListingIterator(Path p, PathFilter filter, boolean needLocation) throws IOException { this.p = p; this.src = getPathName(p); this.filter = filter; this.needLocation = needLocation; // fetch the first batch of entries in the directory thisListing = dfs.listPaths(src, HdfsFileStatus.EMPTY_NAME, needLocation); statistics.incrementReadOps(1); if (thisListing == null) { // the directory does not exist throw new FileNotFoundException("File " + p + " does not exist."); } i = 0; }
private DirListingIterator(Path p, PathFilter filter, boolean needLocation) throws IOException { this.p = p; this.src = getPathName(p); this.filter = filter; this.needLocation = needLocation; // fetch the first batch of entries in the directory thisListing = dfs.listPaths(src, HdfsFileStatus.EMPTY_NAME, needLocation); statistics.incrementReadOps(1); if (thisListing == null) { // the directory does not exist throw new FileNotFoundException("File " + p + " does not exist."); } i = 0; }
public FileStatus[] listStatus(Path p) throws IOException { FileStatus[] infos = dfs.listPaths(getPathName(p)); if (infos == null) return null; for (int i = 0; i < infos.length; i++) { infos[i].makeQualified(this); } return infos; }
public FileStatus[] listStatus(Path p) throws IOException { FileStatus[] infos = dfs.listPaths(getPathName(p)); if (infos == null) return null; FileStatus[] stats = new FileStatus[infos.length]; for (int i = 0; i < infos.length; i++) { stats[i] = makeQualified(infos[i]); } return stats; }
@Override public boolean hasNext() throws IOException { if (thisListing == null) { return false; } if (i>=thisListing.getPartialListing().length && thisListing.hasMore()) { // current listing is exhausted & fetch a new listing thisListing = dfs.listPaths(src, thisListing.getLastName(), needLocation); if (thisListing == null) { throw new FileNotFoundException("File " + src + " does not exist."); } i = 0; } return (i<thisListing.getPartialListing().length); }
/** Check if there is a next item before applying the given filter */ private boolean hasNextNoFilter() throws IOException { if (thisListing == null) { return false; } if (i >= thisListing.getPartialListing().length && thisListing.hasMore()) { // current listing is exhausted & fetch a new listing thisListing = dfs.listPaths(src, thisListing.getLastName(), needLocation); statistics.incrementReadOps(1); if (thisListing == null) { throw new FileNotFoundException("File " + p + " does not exist."); } i = 0; } return (i < thisListing.getPartialListing().length); }
@Override public boolean hasNext() throws IOException { if (thisListing == null) { return false; } if (i>=thisListing.getPartialListing().length && thisListing.hasMore()) { // current listing is exhausted & fetch a new listing thisListing = dfs.listPaths(src, thisListing.getLastName(), needLocation); if (thisListing == null) { return false; // the directory is deleted } i = 0; } return (i<thisListing.getPartialListing().length); }
@Override public boolean hasNext() throws IOException { if (thisListing == null) { return false; } if (i>=thisListing.getPartialListing().length && thisListing.hasMore()) { // current listing is exhausted & fetch a new listing thisListing = dfs.listPaths(src, thisListing.getLastName(), needLocation); if (thisListing == null) { return false; // the directory is deleted } i = 0; } return (i<thisListing.getPartialListing().length); }
void verifyFile(final Path file, final Byte expectedPolicyId) throws Exception { final Path parent = file.getParent(); DirectoryListing children = dfs.getClient().listPaths( parent.toString(), HdfsFileStatus.EMPTY_NAME, true); for (HdfsFileStatus child : children.getPartialListing()) { if (child.getLocalName().equals(file.getName())) { verifyFile(parent, child, expectedPolicyId); return; } } Assert.fail("File " + file + " not found."); }
private void verifyRecursively(final Path parent, final HdfsFileStatus status) throws Exception { if (status.isDir()) { Path fullPath = parent == null ? new Path("/") : status.getFullPath(parent); DirectoryListing children = dfs.getClient().listPaths( fullPath.toString(), HdfsFileStatus.EMPTY_NAME, true); for (HdfsFileStatus child : children.getPartialListing()) { verifyRecursively(fullPath, child); } } else if (!status.isSymlink()) { // is file verifyFile(parent, status, null); } }