@Override public boolean isDirectory() { return myFs.isDirectory(); }
@Override protected LocatedFileStatus computeNext() { while (true) { while (remoteIterator.hasNext()) { LocatedFileStatus status = getLocatedFileStatus(remoteIterator); // Ignore hidden files and directories. Hive ignores files starting with _ and . as well. String fileName = status.getPath().getName(); if (fileName.startsWith("_") || fileName.startsWith(".")) { continue; } if (status.isDirectory()) { switch (nestedDirectoryPolicy) { case IGNORED: continue; case RECURSE: paths.add(status.getPath()); continue; case FAIL: throw new NestedDirectoryNotAllowedException(); } } return status; } if (paths.isEmpty()) { return endOfData(); } remoteIterator = getLocatedFileStatusRemoteIterator(paths.removeFirst()); } }
while (iter.hasNext()) { LocatedFileStatus file = iter.next(); if (file.isDirectory()) {
LocatedFileStatus lfs = it.next(); if (!lfs.isDirectory()) { continue;
public static List<String> getFiles(FileSystem fs, Path rootDir, List<String> files, PathFilter filter) throws IOException { RemoteIterator<LocatedFileStatus> it = fs.listFiles(rootDir, true); while (it.hasNext()) { LocatedFileStatus lfs = it.next(); if (lfs.isDirectory()) { continue; } // apply filter if (filter.accept(lfs.getPath())) { files.add(lfs.getPath().toString()); } } return files; }
if (lfs.isDirectory()) continue; files.add(lfs.getPath());
while (it.hasNext()) { LocatedFileStatus status = it.next(); if (status.isDirectory()) continue;
while (iter.hasNext()) { LocatedFileStatus lfs = iter.next(); if (lfs.isDirectory()) { String msg = Warehouse.getQualifiedName(t) + " is bucketed and has a subdirectory: " + lfs.getPath(); while (iter.hasNext()) { LocatedFileStatus lfs = iter.next(); if(lfs.isDirectory()) { continue;
/** * Will assert that actual files match expected. * @param expectedFiles - suffixes of expected Paths. Must be the same length * @param rootPath - table or partition root where to start looking for actual files, recursively */ void assertExpectedFileSet(Set<String> expectedFiles, String rootPath) throws Exception { int suffixLength = 0; for(String s : expectedFiles) { if(suffixLength > 0) { assert suffixLength == s.length() : "all entries must be the same length. current: " + s; } suffixLength = s.length(); } FileSystem fs = FileSystem.get(hiveConf); Set<String> actualFiles = new HashSet<>(); RemoteIterator<LocatedFileStatus> remoteIterator = fs.listFiles(new Path(rootPath), true); while (remoteIterator.hasNext()) { LocatedFileStatus lfs = remoteIterator.next(); if(!lfs.isDirectory() && org.apache.hadoop.hive.common.FileUtils.HIDDEN_FILES_PATH_FILTER.accept(lfs.getPath())) { String p = lfs.getPath().toString(); actualFiles.add(p.substring(p.length() - suffixLength, p.length())); } } Assert.assertEquals("Unexpected file list", expectedFiles, actualFiles); } void checkExpected(List<String> rs, String[][] expected, String msg, Logger LOG, boolean checkFileName) {
@Override @SuppressWarnings("deprecation") public boolean isDir() { return myFs.isDirectory(); }
@Override @SuppressWarnings("deprecation") public boolean isDir() { return myFs.isDirectory(); }
@Override public boolean isDirectory() { return myFs.isDirectory(); }
@Override @SuppressWarnings("deprecation") public boolean isDir() { return myFs.isDirectory(); }
@Override @SuppressWarnings("deprecation") public boolean isDir() { return myFs.isDirectory(); }
@Override public boolean isDirectory() { return myFs.isDirectory(); }
@Override public boolean isDirectory() { return myFs.isDirectory(); }
@Override public boolean isDirectory() { return myFs.isDirectory(); }
public static void addQueryRecursively(Configuration conf, Path path, boolean sparqlUpdate, int stage) throws IOException { RemoteIterator<LocatedFileStatus> iter = path.getFileSystem(conf).listLocatedStatus(path); while (iter.hasNext()) { LocatedFileStatus stat = iter.next(); if (stat.isDirectory()) { addQueryRecursively(conf, stat.getPath(), sparqlUpdate, stage); } else { addQuery(conf, stat, sparqlUpdate, stage); } } }
private int countWALsInFS(MiniAccumuloClusterImpl cluster) throws Exception { FileSystem fs = cluster.getFileSystem(); RemoteIterator<LocatedFileStatus> iterator = fs .listFiles(new Path(cluster.getConfig().getAccumuloDir() + "/wal"), true); int result = 0; while (iterator.hasNext()) { LocatedFileStatus next = iterator.next(); if (!next.isDirectory()) { result++; } } return result; } }
public void lsR(final String description, final FileSystem fs, final Path dir) throws IOException { RemoteIterator<LocatedFileStatus> files = fs.listFiles(dir, true); LOG.info("{}: {}:", description, dir); StringBuilder sb = new StringBuilder(); while(files.hasNext()) { LocatedFileStatus status = files.next(); sb.append(String.format(" %s; type=%s; length=%d", status.getPath(), status.isDirectory()? "dir" : "file", status.getLen())); } LOG.info("{}", sb); }