Refine search
private List<Path> getFilesRecursively(String fileBackupDir) throws IllegalArgumentException, IOException { FileSystem fs = FileSystem.get((new Path(fileBackupDir)).toUri(), new Configuration()); List<Path> list = new ArrayList<>(); RemoteIterator<LocatedFileStatus> it = fs.listFiles(new Path(fileBackupDir), true); while (it.hasNext()) { Path p = it.next().getPath(); if (HFile.isHFileFormat(fs, p)) { list.add(p); } } return list; }
private long getPathLength(JobConf conf, Path path, Class<? extends InputFormat> clazz, long threshold) throws IOException { if (ContentSummaryInputFormat.class.isAssignableFrom(clazz)) { InputFormat input = HiveInputFormat.getInputFormatFromCache(clazz, conf); return ((ContentSummaryInputFormat)input).getContentSummary(path, conf).getLength(); } else { FileSystem fs = path.getFileSystem(conf); try { long length = 0; RemoteIterator<LocatedFileStatus> results = fs.listFiles(path, true); // No need to iterate more, when threshold is reached // (beneficial especially for object stores) while (length <= threshold && results.hasNext()) { length += results.next().getLen(); } LOG.trace("length=" + length + ", threshold=" + threshold); return length; } catch (FileNotFoundException e) { return 0; } } } }
@Override public BlockLocation[] getLocations(FileSystem fs, FileStatus status) throws IOException { if (status instanceof LocatedFileStatus) { return ((LocatedFileStatus) status).getBlockLocations(); } else { return fs.getFileBlockLocations(status, 0, status.getLen()); } }
private Optional<InternalHiveSplit> createInternalHiveSplit(LocatedFileStatus status, OptionalInt bucketNumber, boolean splittable) { splittable = splittable && isSplittable(inputFormat, fileSystem, status.getPath()); return createInternalHiveSplit( status.getPath(), status.getBlockLocations(), 0, status.getLen(), status.getLen(), bucketNumber, splittable); }
/** * Process the input stat. * If it is a file, return the file stat. * If it is a directory, traverse the directory if recursive is true; * ignore it if recursive is false. * @param stat input status * @throws IOException if any IO error occurs */ private void handleFileStat(LocatedFileStatus stat) throws IOException { if (stat.isFile()) { // file curFile = stat; } else if (recursive) { // directory itors.push(curItor); curItor = listLocatedStatus(stat.getPath()); } }
public static List<String> getFiles(FileSystem fs, Path rootDir, List<String> files, PathFilter filter) throws IOException { RemoteIterator<LocatedFileStatus> it = fs.listFiles(rootDir, true); while (it.hasNext()) { LocatedFileStatus lfs = it.next(); if (lfs.isDirectory()) { continue; } // apply filter if (filter.accept(lfs.getPath())) { files.add(lfs.getPath().toString()); } } return files; }
static SortedSet<byte []> readKeysToSearch(final Configuration conf) throws IOException, InterruptedException { Path keysInputDir = new Path(conf.get(SEARCHER_INPUTDIR_KEY)); FileSystem fs = FileSystem.get(conf); SortedSet<byte []> result = new TreeSet<>(Bytes.BYTES_COMPARATOR); if (!fs.exists(keysInputDir)) { throw new FileNotFoundException(keysInputDir.toString()); } if (!fs.isDirectory(keysInputDir)) { throw new UnsupportedOperationException("TODO"); } else { RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(keysInputDir, false); while(iterator.hasNext()) { LocatedFileStatus keyFileStatus = iterator.next(); // Skip "_SUCCESS" file. if (keyFileStatus.getPath().getName().startsWith("_")) continue; result.addAll(readFileToSearch(conf, fs, keyFileStatus)); } } return result; }
@Override public void killOlderThan(long timestamp) throws IOException { Path taskLogDir = new Path(config.getDirectory()); FileSystem fs = taskLogDir.getFileSystem(hadoopConfig); if (fs.exists(taskLogDir)) { if (!fs.isDirectory(taskLogDir)) { throw new IOE("taskLogDir [%s] must be a directory.", taskLogDir); } RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(taskLogDir); while (iter.hasNext()) { LocatedFileStatus file = iter.next(); if (file.getModificationTime() < timestamp) { Path p = file.getPath(); log.info("Deleting hdfs task log [%s].", p.toUri().toString()); fs.delete(p, true); } if (Thread.currentThread().isInterrupted()) { throw new IOException( new InterruptedException("Thread interrupted. Couldn't delete all tasklogs.") ); } } } } }
while (iter.hasNext()) { LocatedFileStatus lfs = iter.next(); if (lfs.isDirectory()) { String msg = Warehouse.getQualifiedName(t) + " is bucketed and has a subdirectory: " + lfs.getPath(); LOG.error(msg); throw new IllegalStateException(msg); AcidUtils.BucketMetaData bmd = AcidUtils.BucketMetaData.parse(lfs.getPath()); if (bmd.bucketId < 0) { lfs.getPath() + " with non-standard name"; LOG.error(msg); throw new IllegalArgumentException(msg); if (bmd.bucketId > BucketCodec.MAX_BUCKET_ID) { String msg = "Bucketed table " + Warehouse.getQualifiedName(t) + " contains file " + lfs.getPath() + " with bucketId=" + bmd.bucketId + " that is out of range"; LOG.error(msg); throw new IllegalArgumentException(msg); deltaToFileMap.computeIfAbsent(bmd.copyNumber, makeList).add(lfs.getPath()); while (iter.hasNext()) { LocatedFileStatus lfs = iter.next(); if(lfs.isDirectory()) { continue; AcidUtils.BucketMetaData bmd = AcidUtils.BucketMetaData.parse(lfs.getPath());
private void enqueue(String path) throws IOException { Path filePath = new Path(path); if (!fs.exists(filePath) || fs.getFileStatus(filePath) == null) { log.info("Cannot enqueue file {} because it does not exist but got an event from the FS", filePath.toString()); return; } RemoteIterator<LocatedFileStatus> it = fs.listFiles(filePath, false); while (it.hasNext()) { LocatedFileStatus status = it.next(); if (!status.isFile() || !fileRegexp.matcher(status.getPath().getName()).find()) continue; fileQueue.offer(toMetadata(status)); } } }
private void cacheMetadata(FileMetadataExprType type, String location) throws MetaException, IOException, InterruptedException { Path path = new Path(location); FileSystem fs = path.getFileSystem(conf); List<Path> files; if (!fs.isDirectory(path)) { files = Lists.newArrayList(path); } else { files = new ArrayList<>(); RemoteIterator<LocatedFileStatus> iter = fs.listFiles(path, true); while (iter.hasNext()) { LocatedFileStatus lfs = iter.next(); if (lfs.isDirectory()) continue; files.add(lfs.getPath()); fileId = HdfsUtils.getFileId(fs, Path.getPathWithoutSchemeAndAuthority(file).toString()); } catch (UnsupportedOperationException ex) { LOG.error("Cannot cache file metadata for " + location + "; "
@Override public String[] listFiles(URI fileUri, boolean recursive) throws IOException { ArrayList<String> filePathStrings = new ArrayList<>(); Path path = new Path(fileUri); if (_hadoopFS.exists(path)) { RemoteIterator<LocatedFileStatus> fileListItr = _hadoopFS.listFiles(path, recursive); while (fileListItr != null && fileListItr.hasNext()) { LocatedFileStatus file = fileListItr.next(); filePathStrings.add(file.getPath().toUri().toString()); } } else { throw new IllegalArgumentException("segmentUri is not valid"); } String[] retArray = new String[filePathStrings.size()]; filePathStrings.toArray(retArray); return retArray; }
jobConf.setWorkingDirectory(new Path(converterConfig.getDistributedSuccessCache())); final FileSystem fs = jobDir.getFileSystem(job.getConfiguration()); final RemoteIterator<LocatedFileStatus> it = fs.listFiles(jobDir, true); final List<Path> goodPaths = new ArrayList<>(); while (it.hasNext()) { final LocatedFileStatus locatedFileStatus = it.next(); if (locatedFileStatus.isFile()) { final Path myPath = locatedFileStatus.getPath(); if (ConvertingOutputFormat.DATA_SUCCESS_KEY.equals(myPath.getName())) { goodPaths.add(new Path(myPath.getParent(), ConvertingOutputFormat.DATA_FILE_KEY));
public static BackupInfo loadBackupInfo(Path backupRootPath, String backupId, FileSystem fs) throws IOException { Path backupPath = new Path(backupRootPath, backupId); RemoteIterator<LocatedFileStatus> it = fs.listFiles(backupPath, true); while (it.hasNext()) { LocatedFileStatus lfs = it.next(); if (lfs.getPath().getName().equals(BackupManifest.MANIFEST_FILE_NAME)) { // Load BackupManifest BackupManifest manifest = new BackupManifest(fs, lfs.getPath().getParent()); BackupInfo info = manifest.toBackupInfo(); return info; } } return null; }
Visitor visitor) throws IOException { Path p = getRealHDFSPath(folderPath); if (!fs.exists(p) || !fs.isDirectory(p)) { return; String fsPathPrefix = p.toUri().getPath(); String resPathPrefix = folderPath.endsWith("/") ? folderPath : folderPath + "/"; RemoteIterator<LocatedFileStatus> it = fs.listFiles(p, recursive); while (it.hasNext()) { LocatedFileStatus status = it.next(); if (status.isDirectory()) continue; String path = status.getPath().toUri().getPath(); if (!path.startsWith(fsPathPrefix)) throw new IllegalStateException("File path " + path + " is supposed to start with " + fsPathPrefix); if (filter.matches(resPath, status.getModificationTime())) { RawResource raw; if (loadContent) raw = new RawResource(resPath, status.getModificationTime(), fs.open(status.getPath())); else raw = new RawResource(resPath, status.getModificationTime());
FileSystem fs = FileSystem.get(backupRootPath.toUri(), conf); RemoteIterator<LocatedFileStatus> it = fs.listLocatedStatus(backupRootPath); while (it.hasNext()) { LocatedFileStatus lfs = it.next(); if (!lfs.isDirectory()) { continue; String backupId = lfs.getPath().getName(); try { BackupInfo info = loadBackupInfo(backupRootPath, backupId, fs); infos.add(info); } catch (IOException e) { LOG.error("Can not load backup info from: " + lfs.getPath(), e);
private List<FileStatus> getListOfWALFiles(Configuration c) throws IOException { Path logRoot = new Path(CommonFSUtils.getWALRootDir(c), HConstants.HREGION_LOGDIR_NAME); FileSystem fs = logRoot.getFileSystem(c); RemoteIterator<LocatedFileStatus> it = fs.listFiles(logRoot, true); List<FileStatus> logFiles = new ArrayList<FileStatus>(); while (it.hasNext()) { LocatedFileStatus lfs = it.next(); if (lfs.isFile() && !AbstractFSWALProvider.isMetaFile(lfs.getPath())) { logFiles.add(lfs); LOG.info(Objects.toString(lfs)); } } return logFiles; }
RemoteIterator<LocatedFileStatus> rit = fs.listFiles(pt, false); Vector<FSDataInputStream> fileHandleList = new Vector<FSDataInputStream>(); while (rit.hasNext()) { Path path = rit.next().getPath(); if (path.getName().startsWith("part-")) { long fileSize = fs.getFileStatus(path).getLen(); if (offset < fileSize) { FSDataInputStream inputStream = fs.open(path); if (offset > 0) { inputStream.seek(offset);
private List<String> getCommitFiles(final String basePath, final FileSystem fs) throws IOException { final List<String> commitFiles = new ArrayList<>(); final RemoteIterator<LocatedFileStatus> filesI = fs.listFiles(new Path(basePath), true); while (filesI.hasNext()) { final LocatedFileStatus fileStatus = filesI.next(); if (fileStatus.isFile() && fileStatus.getPath().getName().endsWith("commit")) { commitFiles.add(fileStatus.getPath().getName()); } } return commitFiles; } }
catName, table.getDbName()), table); } else { tablePath = wh.getDnsPath(new Path(table.getSd().getLocation())); new FileUtils.RemoteIteratorWithFilter(fs.listFiles(tablePath, true), FileUtils .RemoteIteratorWithFilter.HIDDEN_FILES_FULL_PATH_FILTER); while (iterator.hasNext()) { LocatedFileStatus fileStatus = iterator.next(); if (!fileStatus.isFile()) { continue; (ORIGINAL_PATTERN.matcher(fileStatus.getPath().getName()).matches() || ORIGINAL_PATTERN_COPY.matcher(fileStatus.getPath().getName()).matches() ); if (!validFile) { throw new IllegalStateException("Unexpected data file name format. Cannot convert " + Warehouse.getQualifiedName(table) + " to transactional table. File: " + fileStatus.getPath());