Refine search
private List<Path> getFilesRecursively(String fileBackupDir) throws IllegalArgumentException, IOException { FileSystem fs = FileSystem.get((new Path(fileBackupDir)).toUri(), new Configuration()); List<Path> list = new ArrayList<>(); RemoteIterator<LocatedFileStatus> it = fs.listFiles(new Path(fileBackupDir), true); while (it.hasNext()) { Path p = it.next().getPath(); if (HFile.isHFileFormat(fs, p)) { list.add(p); } } return list; }
@Override public FileStatus[] listStatus(Path path) throws IOException { STATS.newListStatusCall(); List<LocatedFileStatus> list = new ArrayList<>(); RemoteIterator<LocatedFileStatus> iterator = listLocatedStatus(path); while (iterator.hasNext()) { list.add(iterator.next()); } return toArray(list, LocatedFileStatus.class); }
private void deleteAppLogDir(FileContext fs, DeletionService del, String logDir) throws IOException { RemoteIterator<FileStatus> fileStatuses = fs.listStatus(new Path(logDir)); if (fileStatuses != null) { while (fileStatuses.hasNext()) { FileStatus fileStatus = fileStatuses.next(); String appName = fileStatus.getPath().getName(); if (appName.matches("^application_\\d+_\\d+_DEL_\\d+$")) { LOG.info("delete app log dir," + appName); del.delete(null, fileStatus.getPath()); } } } }
final Path workingDirPath = new Path(workingDirPathStr); FileSystem fs; try { fs = workingDirPath.getFileSystem(config); log.info(fs.getScheme()); log.info("FileSystem URI:" + fs.getUri()); if (!fs.exists(workingDirPath)) { throw new SegmentLoadingException("Working directory [%s] doesn't exist.", workingDirPath); final RemoteIterator<LocatedFileStatus> it = fs.listFiles(workingDirPath, true); while (it.hasNext()) { final LocatedFileStatus locatedFileStatus = it.next(); final Path path = locatedFileStatus.getPath(); if (path.getName().endsWith("descriptor.json")) { timestampedSegments, dataSegment, locatedFileStatus.getModificationTime() ); } else {
static SortedSet<byte []> readKeysToSearch(final Configuration conf) throws IOException, InterruptedException { Path keysInputDir = new Path(conf.get(SEARCHER_INPUTDIR_KEY)); FileSystem fs = FileSystem.get(conf); SortedSet<byte []> result = new TreeSet<>(Bytes.BYTES_COMPARATOR); if (!fs.exists(keysInputDir)) { throw new FileNotFoundException(keysInputDir.toString()); } if (!fs.isDirectory(keysInputDir)) { throw new UnsupportedOperationException("TODO"); } else { RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(keysInputDir, false); while(iterator.hasNext()) { LocatedFileStatus keyFileStatus = iterator.next(); // Skip "_SUCCESS" file. if (keyFileStatus.getPath().getName().startsWith("_")) continue; result.addAll(readFileToSearch(conf, fs, keyFileStatus)); } } return result; }
assertFalse(targetFileSystem.exists(targetDir)); srcPath = new Path("file://" + srcDir.getAbsolutePath()); } else { srcPath = new Path(srcDir.getAbsolutePath()); HashMap<String, LocalResource> localResources = new HashMap<>(); AbstractYarnClusterDescriptor.uploadAndRegisterFiles( Collections.singletonList(new File(srcPath.toUri().getPath())), targetFileSystem, targetDir, targetFileSystem.listFiles(workDir, true); HashMap<String /* (relative) path */, /* contents */ String> targetFiles = new HashMap<>(4); workDir.toString().length() + 1; // one more for the concluding "/" while (targetFilesIterator.hasNext()) { LocatedFileStatus targetFile = targetFilesIterator.next(); try (FSDataInputStream in = targetFileSystem.open(targetFile.getPath())) { String absolutePathString = targetFile.getPath().toString(); String relativePath = absolutePathString.substring(workDirPrefixLength); targetFiles.put(relativePath, in.readUTF());
@Override public void killOlderThan(long timestamp) throws IOException { Path taskLogDir = new Path(config.getDirectory()); FileSystem fs = taskLogDir.getFileSystem(hadoopConfig); if (fs.exists(taskLogDir)) { if (!fs.isDirectory(taskLogDir)) { throw new IOE("taskLogDir [%s] must be a directory.", taskLogDir); } RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(taskLogDir); while (iter.hasNext()) { LocatedFileStatus file = iter.next(); if (file.getModificationTime() < timestamp) { Path p = file.getPath(); log.info("Deleting hdfs task log [%s].", p.toUri().toString()); fs.delete(p, true); } if (Thread.currentThread().isInterrupted()) { throw new IOException( new InterruptedException("Thread interrupted. Couldn't delete all tasklogs.") ); } } } } }
/** Test the FileStatus obtained calling listStatus on a file */ @Test public void testListStatusOnFile() throws IOException { FileStatus[] stats = fs.listStatus(file1); assertEquals(1, stats.length); FileStatus status = stats[0]; assertFalse(file1 + " should be a file", status.isDirectory()); assertEquals(blockSize, status.getBlockSize()); assertEquals(1, status.getReplication()); assertEquals(fileSize, status.getLen()); assertEquals(file1.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString(), status.getPath().toString()); RemoteIterator<FileStatus> itor = fc.listStatus(file1); status = itor.next(); assertEquals(stats[0], status); assertFalse(file1 + " should be a file", status.isDirectory()); }
final FileSystem fs = path.getFileSystem(config); if (fs.isDirectory(path)) { return RetryUtils.retry( () -> { if (!fs.exists(path)) { throw new SegmentLoadingException("No files found at [%s]", path.toString()); final RemoteIterator<LocatedFileStatus> children = fs.listFiles(path, false); final FileUtils.FileCopyResult result = new FileUtils.FileCopyResult(); while (children.hasNext()) { final LocatedFileStatus child = children.next(); final Path childPath = child.getPath(); final String fname = childPath.getName(); if (fs.isDirectory(childPath)) { log.warn("[%s] is a child directory, skipping", childPath.toString());
private void cacheMetadata(FileMetadataExprType type, String location) throws MetaException, IOException, InterruptedException { Path path = new Path(location); FileSystem fs = path.getFileSystem(conf); List<Path> files; if (!fs.isDirectory(path)) { files = Lists.newArrayList(path); } else { files = new ArrayList<>(); RemoteIterator<LocatedFileStatus> iter = fs.listFiles(path, true); while (iter.hasNext()) { LocatedFileStatus lfs = iter.next(); if (lfs.isDirectory()) continue; files.add(lfs.getPath()); fileId = HdfsUtils.getFileId(fs, Path.getPathWithoutSchemeAndAuthority(file).toString()); } catch (UnsupportedOperationException ex) { LOG.error("Cannot cache file metadata for " + location + "; "
@Override public String[] listFiles(URI fileUri, boolean recursive) throws IOException { ArrayList<String> filePathStrings = new ArrayList<>(); Path path = new Path(fileUri); if (_hadoopFS.exists(path)) { RemoteIterator<LocatedFileStatus> fileListItr = _hadoopFS.listFiles(path, recursive); while (fileListItr != null && fileListItr.hasNext()) { LocatedFileStatus file = fileListItr.next(); filePathStrings.add(file.getPath().toUri().toString()); } } else { throw new IllegalArgumentException("segmentUri is not valid"); } String[] retArray = new String[filePathStrings.size()]; filePathStrings.toArray(retArray); return retArray; }
public static List<String> listHDFSDirectory(String directory, boolean doIncludeFiles, boolean doIncludeSubdirectories) throws IOException { Configuration config = getConfiguration(); FileSystem fileSystem = FileSystem.get(config); List<String> files = new ArrayList<String>(); RemoteIterator<LocatedFileStatus> fileStatusListIterator= fileSystem.listLocatedStatus(new Path(directory)); while(fileStatusListIterator.hasNext()) { FileStatus fileStatus = fileStatusListIterator.next(); if(doIncludeFiles) { files.add(fileStatus.getPath().toString()); } else { if(fileStatus.isDirectory()) { files.add(fileStatus.getPath().toString()); } } } return files; }
protected void doInitialize() { FileSystem fs = CommonUtils.openHdfsConnect(); try { if (train) { hdfsIter= fs.listFiles(new Path(CommonUtils.TRAIN_HDFS_PATH), true); } else { hdfsIter= fs.listFiles(new Path(CommonUtils.VALIDATE_HDFS_PATH), true); } while (hdfsIter.hasNext()) { LocatedFileStatus next = hdfsIter.next(); Path path = next.getPath(); String currentPath = path.toUri().getPath(); fileNames.add(path.toString()); String name = FilenameUtils.getBaseName((new File(currentPath)).getParent()); if (!labels.contains(name)) { labels.add(name); } } Collections.shuffle(fileNames); fileIterator = fileNames.iterator(); numExample = fileNames.size(); } catch (Exception e) { throw new RuntimeException(e); } finally { CommonUtils.closeHdfsConnect(fs); } } public DataSet convertDataSet(int num) {
@Override protected LocatedFileStatus[] listLocatedStatus(JobConf job) throws IOException { LocatedFileStatus[] files = super.listLocatedStatus(job); for (int i = 0; i < files.length; i++) { FileStatus file = files[i]; if (file.isDir()) { // it's a MapFile Path dataFile = new Path(file.getPath(), MapFile.DATA_FILE_NAME); FileSystem fs = file.getPath().getFileSystem(job); // use the data file files[i] = fs.listLocatedStatus(dataFile).next(); } } return files; }
public static BackupInfo loadBackupInfo(Path backupRootPath, String backupId, FileSystem fs) throws IOException { Path backupPath = new Path(backupRootPath, backupId); RemoteIterator<LocatedFileStatus> it = fs.listFiles(backupPath, true); while (it.hasNext()) { LocatedFileStatus lfs = it.next(); if (lfs.getPath().getName().equals(BackupManifest.MANIFEST_FILE_NAME)) { // Load BackupManifest BackupManifest manifest = new BackupManifest(fs, lfs.getPath().getParent()); BackupInfo info = manifest.toBackupInfo(); return info; } } return null; }
jobConf.setWorkingDirectory(new Path(converterConfig.getDistributedSuccessCache())); final FileSystem fs = jobDir.getFileSystem(job.getConfiguration()); final RemoteIterator<LocatedFileStatus> it = fs.listFiles(jobDir, true); final List<Path> goodPaths = new ArrayList<>(); while (it.hasNext()) { final LocatedFileStatus locatedFileStatus = it.next(); if (locatedFileStatus.isFile()) { final Path myPath = locatedFileStatus.getPath(); if (ConvertingOutputFormat.DATA_SUCCESS_KEY.equals(myPath.getName())) { goodPaths.add(new Path(myPath.getParent(), ConvertingOutputFormat.DATA_FILE_KEY));
String lastRelDir = null; HashSet<Path> results = new HashSet<Path>(); String relRoot = Path.getPathWithoutSchemeAndAuthority(path).toString(); if (!relRoot.endsWith(Path.SEPARATOR)) { relRoot += Path.SEPARATOR; RemoteIterator<LocatedFileStatus> allFiles = fs.listFiles(path, true); while (allFiles.hasNext()) { LocatedFileStatus lfs = allFiles.next(); Path lfsPath = lfs.getPath(); Path dirPath = Path.getPathWithoutSchemeAndAuthority(lfsPath); String dir = dirPath.toString(); if (!dir.startsWith(relRoot)) { continue; results.add(fs.makeQualified(candidate));
public static File copyDirectoryIfNonExistent(final FileSystem fileSystem, final String directory) { try { final String hadoopGremlinLibsRemote = "hadoop-gremlin-" + Gremlin.version() + "-libs"; final Path path = new Path(directory); if (Boolean.valueOf(System.getProperty("is.testing", "false")) || (fileSystem.exists(path) && fileSystem.isDirectory(path))) { final File tempDirectory = new File(System.getProperty("java.io.tmpdir") + File.separator + hadoopGremlinLibsRemote); assert tempDirectory.exists() || tempDirectory.mkdirs(); final String tempPath = tempDirectory.getAbsolutePath() + File.separator + path.getName(); final RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(path, false); while (files.hasNext()) { final LocatedFileStatus f = files.next(); fileSystem.copyToLocalFile(false, f.getPath(), new Path(tempPath + System.getProperty("file.separator") + f.getPath().getName()), true); } return new File(tempPath); } else return new File(directory); } catch (final IOException e) { throw new IllegalStateException(e.getMessage(), e); } } }