@Override public boolean hasNext() throws IOException { return orig.hasNext(); }
@Override public boolean hasNext() throws IOException { return stats.hasNext(); }
@Override public boolean hasNext() throws IOException { return innerIter.hasNext(); }
@Override public boolean hasNext() throws IOException { return itor.hasNext(); }
@Override public boolean hasNext() { try { return fileStatusIterator.hasNext(); } catch (IOException e) { throw processException(e); } }
@Override public FileStatus[] listStatus(Path path) throws IOException { STATS.newListStatusCall(); List<LocatedFileStatus> list = new ArrayList<>(); RemoteIterator<LocatedFileStatus> iterator = listLocatedStatus(path); while (iterator.hasNext()) { list.add(iterator.next()); } return toArray(list, LocatedFileStatus.class); }
TreeSet<String> getAllFilePath(Path filePath, String resPathPrefix) throws IOException { String fsPathPrefix = filePath.toUri().getPath(); TreeSet<String> fileList = new TreeSet<>(); RemoteIterator<LocatedFileStatus> it = fs.listFiles(filePath, true); while (it.hasNext()) { String path = it.next().getPath().toUri().getPath(); if (!path.startsWith(fsPathPrefix)) throw new IllegalStateException("File path " + path + " is supposed to start with " + fsPathPrefix); String resPath = resPathPrefix + path.substring(fsPathPrefix.length() + 1); fileList.add(resPath); } return fileList; }
private List<Path> getFilesRecursively(String fileBackupDir) throws IllegalArgumentException, IOException { FileSystem fs = FileSystem.get((new Path(fileBackupDir)).toUri(), new Configuration()); List<Path> list = new ArrayList<>(); RemoteIterator<LocatedFileStatus> it = fs.listFiles(new Path(fileBackupDir), true); while (it.hasNext()) { Path p = it.next().getPath(); if (HFile.isHFileFormat(fs, p)) { list.add(p); } } return list; }
@Override public boolean hasNext() throws IOException { while (curFile == null) { if (curItor.hasNext()) { handleFileStat(curItor.next()); } else if (!itors.empty()) { curItor = itors.pop(); } else { return false; } } return true; }
public static void main(String[] args) throws IOException{ DatasetReaderFromHdfs ds = new DatasetReaderFromHdfs(); int j = 0; while (hdfsIter.hasNext()) { LocatedFileStatus next = hdfsIter.next(); Path path = next.getPath(); String currentPath = path.toUri().getPath(); //String index = getRelativeFilename(currentPath); System.out.println("file name : i = " + j ++ + " path=" + currentPath); } }
public static List<FileStatus> listLocatedStatus(final FileSystem fs, final Path path, final PathFilter filter ) throws IOException { RemoteIterator<LocatedFileStatus> itr = fs.listLocatedStatus(path); List<FileStatus> result = new ArrayList<FileStatus>(); while(itr.hasNext()) { FileStatus stat = itr.next(); if (filter == null || filter.accept(stat.getPath())) { result.add(stat); } } return result; }
@Override public void killOlderThan(long timestamp) throws IOException { Path taskLogDir = new Path(config.getDirectory()); FileSystem fs = taskLogDir.getFileSystem(hadoopConfig); if (fs.exists(taskLogDir)) { if (!fs.isDirectory(taskLogDir)) { throw new IOE("taskLogDir [%s] must be a directory.", taskLogDir); } RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(taskLogDir); while (iter.hasNext()) { LocatedFileStatus file = iter.next(); if (file.getModificationTime() < timestamp) { Path p = file.getPath(); log.info("Deleting hdfs task log [%s].", p.toUri().toString()); fs.delete(p, true); } if (Thread.currentThread().isInterrupted()) { throw new IOException( new InterruptedException("Thread interrupted. Couldn't delete all tasklogs.") ); } } } } }
void findNext() throws IOException { while (iter.hasNext()) { LocatedFileStatus status = iter.next(); if (filter.accept(status.getPath())) { nextFile = status; return; } } // No more matching files in the iterator nextFile = null; } }
public static BackupInfo loadBackupInfo(Path backupRootPath, String backupId, FileSystem fs) throws IOException { Path backupPath = new Path(backupRootPath, backupId); RemoteIterator<LocatedFileStatus> it = fs.listFiles(backupPath, true); while (it.hasNext()) { LocatedFileStatus lfs = it.next(); if (lfs.getPath().getName().equals(BackupManifest.MANIFEST_FILE_NAME)) { // Load BackupManifest BackupManifest manifest = new BackupManifest(fs, lfs.getPath().getParent()); BackupInfo info = manifest.toBackupInfo(); return info; } } return null; }
@Override public boolean hasNext() throws IOException { while (next == null && iter.hasNext()) { LocatedFileStatus unfilteredNext = iter.next(); if (!isChecksumFile(unfilteredNext.getPath())) { next = unfilteredNext; } } return next != null; }
public static List<String> getFiles(FileSystem fs, Path rootDir, List<String> files, PathFilter filter) throws IOException { RemoteIterator<LocatedFileStatus> it = fs.listFiles(rootDir, true); while (it.hasNext()) { LocatedFileStatus lfs = it.next(); if (lfs.isDirectory()) { continue; } // apply filter if (filter.accept(lfs.getPath())) { files.add(lfs.getPath().toString()); } } return files; }
protected void dumpBackupDir() throws IOException { // Dump Backup Dir FileSystem fs = FileSystem.get(conf1); RemoteIterator<LocatedFileStatus> it = fs.listFiles(new Path(BACKUP_ROOT_DIR), true); while (it.hasNext()) { LOG.debug(Objects.toString(it.next().getPath())); } } }
private List<String> listDir(Path p) throws IOException { ArrayList<String> result = new ArrayList<>(); RemoteIterator<LocatedFileStatus> fileNames = fs.listFiles(p, false); while (fileNames.hasNext()) { LocatedFileStatus fileStatus = fileNames.next(); result.add(Path.getPathWithoutSchemeAndAuthority(fileStatus.getPath()).toString()); } return result; }
private List<FileStatus> getListOfWALFiles(Configuration c) throws IOException { Path logRoot = new Path(CommonFSUtils.getWALRootDir(c), HConstants.HREGION_LOGDIR_NAME); FileSystem fs = logRoot.getFileSystem(c); RemoteIterator<LocatedFileStatus> it = fs.listFiles(logRoot, true); List<FileStatus> logFiles = new ArrayList<FileStatus>(); while (it.hasNext()) { LocatedFileStatus lfs = it.next(); if (lfs.isFile() && !AbstractFSWALProvider.isMetaFile(lfs.getPath())) { logFiles.add(lfs); LOG.info(Objects.toString(lfs)); } } return logFiles; }
private boolean hasHFileLink(Path tableDir) throws IOException { if (fs.exists(tableDir)) { RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(tableDir, true); while (iterator.hasNext()) { LocatedFileStatus fileStatus = iterator.next(); if (fileStatus.isFile() && HFileLink.isHFileLink(fileStatus.getPath())) { return true; } } } return false; }