Refine search
/** * Utility method that determines if a specified directory already has * contents (non-hidden files) or not - useful to determine if an * immutable table already has contents, for example. * @param fs * @param path * @throws IOException */ public static boolean isDirEmpty(FileSystem fs, Path path) throws IOException { if (fs.exists(path)) { FileStatus[] status = fs.globStatus(new Path(path, "*"), hiddenFileFilter); if (status.length > 0) { return false; } } return true; }
public static List<Path> getTableDirs(final FileSystem fs, final Path rootdir) throws IOException { List<Path> tableDirs = new LinkedList<>(); for(FileStatus status : fs.globStatus(new Path(rootdir, new Path(HConstants.BASE_NAMESPACE_DIR, "*")))) { tableDirs.addAll(FSUtils.getLocalTableDirs(fs, status.getPath())); } return tableDirs; }
public static FileStatus[] matchFilesOrDir(FileSystem fs, Path path) throws IOException { FileStatus[] srcs = fs.globStatus(path, new PathFilter() { @Override public boolean accept(Path p) { String name = p.getName(); return name.equals(EximUtil.METADATA_NAME) || (!name.startsWith("_") && !name.startsWith(".")); } }); if ((srcs != null) && srcs.length == 1) { if (srcs[0].isDirectory()) { srcs = fs.listStatus(srcs[0].getPath(), new PathFilter() { @Override public boolean accept(Path p) { String name = p.getName(); return !name.startsWith("_") && !name.startsWith("."); } }); } } return (srcs); }
/** * Find all users with trash directories by listing all directories in * the file system matching the trash pattern given by {@link #TRASH_LOCATION_KEY}. * @return List of users with trash directory. * @throws IOException */ protected List<String> getAllUsersWithTrash() throws IOException { Path trashLocationGlob = new Path(this.properties.getProperty(TRASH_LOCATION_KEY).replaceAll("\\$USER", "*")); Pattern userPattern = Pattern.compile(this.properties.getProperty(TRASH_LOCATION_KEY).replaceAll("\\$USER", "([^/])")); List<String> users = Lists.newArrayList(); for (FileStatus fileStatus : this.fs.globStatus(trashLocationGlob)) { Matcher matcher = userPattern.matcher(fileStatus.getPath().toString()); if (matcher.find()) { users.add(matcher.group(1)); } } return users; }
private static Path[] getMmDirectoryCandidatesGlobStatus(FileSystem fs, Path path, int skipLevels, PathFilter filter, long writeId, int stmtId, boolean isBaseDir) throws IOException { StringBuilder sb = new StringBuilder(path.toUri().getPath()); for (int i = 0; i < skipLevels; i++) { sb.append(Path.SEPARATOR).append('*'); } if (stmtId < 0) { // Note: this does not work. // sb.append(Path.SEPARATOR).append(AcidUtils.deltaSubdir(writeId, writeId)).append("_*"); throw new AssertionError("GlobStatus should not be called without a statement ID"); } else { sb.append(Path.SEPARATOR).append(AcidUtils.baseOrDeltaSubdir(isBaseDir, writeId, writeId, stmtId)); } Path pathPattern = new Path(path, sb.toString()); return statusToPath(fs.globStatus(pathPattern, filter)); }
public static FileStatus[] matchFilesOrDir(FileSystem fs, Path path) throws IOException { FileStatus[] srcs = fs.globStatus(path, new PathFilter() { @Override public boolean accept(Path p) { String name = p.getName(); return name.equals(EximUtil.METADATA_NAME) ? true : !name.startsWith("_") && !name.startsWith("."); } }); if ((srcs != null) && srcs.length == 1) { if (srcs[0].isDir()) { srcs = fs.listStatus(srcs[0].getPath(), new PathFilter() { @Override public boolean accept(Path p) { String name = p.getName(); return !name.startsWith("_") && !name.startsWith("."); } }); } } return (srcs); }
public void run() throws Exception { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); Path path = new Path(_segmentPath); FileStatus[] fileStatusArr = fs.globStatus(path); for (FileStatus fileStatus : fileStatusArr) { if (fileStatus.isDirectory()) { pushDir(fs, fileStatus.getPath()); } else { pushOneTarFile(fs, fileStatus.getPath()); } } }
@Override public void call(T ignored) throws IOException { Path dataDirPath = new Path(dataDirString + "/*"); FileSystem fs = FileSystem.get(dataDirPath.toUri(), hadoopConf); FileStatus[] inputPathStatuses = fs.globStatus(dataDirPath); if (inputPathStatuses != null) { long oldestTimeAllowed = System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(maxAgeHours, TimeUnit.HOURS); Arrays.stream(inputPathStatuses).filter(FileStatus::isDirectory).map(FileStatus::getPath). filter(subdir -> { Matcher m = dirTimestampPattern.matcher(subdir.getName()); return m.find() && Long.parseLong(m.group(1)) < oldestTimeAllowed; }).forEach(subdir -> { log.info("Deleting old data at {}", subdir); try { fs.delete(subdir, true); } catch (IOException e) { log.warn("Unable to delete {}; continuing", subdir, e); } }); } }
} else { statuses = fs.globStatus(path); if(statuses == null || statuses.length == 0) throw new IllegalArgumentException("No matches for path pattern " + path); if(!HadoopUtils.shouldPathBeIgnored(status.getPath())) { schemas.add(getSchemaFromPath(fs, status.getPath()));
public void run() throws Exception { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); Path path = new Path(_segmentPath); FileStatus[] fileStatusArr = fs.globStatus(path); for (FileStatus fileStatus : fileStatusArr) { if (fileStatus.isDirectory()) { pushDir(fs, fileStatus.getPath()); } else { pushOneTarFile(fs, fileStatus.getPath()); } } }
sb.append(Path.SEPARATOR).append("*"); Path pathPattern = new Path(path, sb.toString()); return Lists.newArrayList(fs.globStatus(pathPattern, FileUtils.HIDDEN_FILES_PATH_FILTER));
try { fs = FileSystem.get(filePath.toUri(), conf); FileStatus[] fileMatches = fs.globStatus(filePath); fileStatus = FileUtils.getPathOrParentThatExists(fs, filePath.getParent()); Path path = fileStatus.getPath(); if (pickParent){ LOG.debug("Checking fs privileges for parent path {} for nonexistent {}",
@SuppressWarnings("nls") public FileStatus[] getSortedPaths() { try { // Previously, this got the filesystem of the Table, which could be // different from the filesystem of the partition. FileSystem fs = FileSystem.get(getPath().toUri(), SessionState.getSessionConf()); String pathPattern = getPath().toString(); if (getNumBuckets() > 0) { pathPattern = pathPattern + "/*"; } LOG.info("Path pattern = " + pathPattern); FileStatus srcs[] = fs.globStatus(new Path(pathPattern), FileUtils.HIDDEN_FILES_PATH_FILTER); Arrays.sort(srcs); for (FileStatus src : srcs) { LOG.info("Got file: " + src.getPath()); } if (srcs.length == 0) { return null; } return srcs; } catch (Exception e) { throw new RuntimeException("Cannot get path ", e); } }
synchronized (parentDirLockSet) { if (!parentDirLockSet.contains(parentDirStr)) { Path parentDir = new Path(parentDirStr); FileSystem fs = FileSystem.getLocal(conf); fs.delete(parentDir, true); // it's ok if the dir doesn't exist now new Path(pathPattern, "*.jar") : pathPattern; // append "*.jar" if a directory is specified FileStatus[] fileStatuses = fs.globStatus(pathPattern1); // return all files that match the pattern if (fileStatuses == null || fileStatuses.length == 0) { // if no one matches throw new FileNotFoundException(pathPattern1.toString()); File dst = new File(parentDirStr, "." + pathPrefix + "." + path.getName() + "." + System.currentTimeMillis() + ".jar"); fs.copyToLocalFile(path, new Path(dst.toString())); dst.deleteOnExit();
dstFs = toPath.getFileSystem(conf); FileStatus[] srcs = srcFs.globStatus(fromPath, new EximPathFilter()); "Recursing into a single child directory {}", srcs[0].getPath().getName()); srcs = srcFs.listStatus(srcs[0].getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER); String oneSrcPathStr = oneSrc.getPath().toString(); console.printInfo("Copying file: " + oneSrcPathStr); Utilities.FILE_OP_LOGGER.debug("Copying file {} to {}", oneSrcPathStr, toPath);
sb.append(Path.SEPARATOR).append("*"); Path pathPattern = new Path(path, sb.toString()); if (!allLevelsBelow) { return Lists.newArrayList(fs.globStatus(pathPattern, filter)); for (FileStatus status : fs.globStatus(pathPattern)) { if (filter.accept(status.getPath())) { results.add(status); for (FileStatus child : fs.listStatus(status.getPath())) { if (filter.accept(child.getPath())) { results.add(child);
@Test public void test() throws IOException { FileSystem fs = mock(FileSystem.class); String datasetPathStr = "/path/to/dataset"; String dataset1 = "datasetVersion1"; String dataset2 = "datasetVersion2"; Path datasetPath = new Path(datasetPathStr); Path globbedPath = new Path(datasetPathStr + "/*"); Path datasetVersion1 = new Path(datasetPathStr + "/" + dataset1); Path datasetVersion2 = new Path(datasetPathStr + "/" + dataset2); when(fs.globStatus(globbedPath)). thenReturn(new FileStatus[]{new FileStatus(0, true, 0, 0, 0, datasetVersion1), new FileStatus(0, true, 0, 0, 0, datasetVersion2)}); DatasetVersionFinder<StringDatasetVersion> versionFinder = new MockDatasetVersionFinder(fs, new Properties()); List<StringDatasetVersion> datasetVersions = Lists.newArrayList(versionFinder.findDatasetVersions(new MockDataset(datasetPath))); Assert.assertEquals(datasetVersions.size(), 2); Assert.assertEquals(datasetVersions.get(0).getVersion(), dataset1); Assert.assertEquals(datasetVersions.get(0).getPathsToDelete().iterator().next(), datasetVersion1); Assert.assertEquals(datasetVersions.get(1).getVersion(), dataset2); Assert.assertEquals(datasetVersions.get(1).getPathsToDelete().iterator().next(), datasetVersion2); }
} else { statuses = fs.globStatus(path); List<JsonSchema> schemas = new ArrayList<JsonSchema>(); for(FileStatus status: statuses) { if(!HadoopUtils.shouldPathBeIgnored(status.getPath())) { if(!checkSameSchema) { return getSchemaFromPath(fs, status.getPath(), checkSameSchema); schemas.add(getSchemaFromPath(fs, status.getPath(), checkSameSchema));
/** * get all paths for this partition in a sorted manner */ @SuppressWarnings("nls") public FileStatus[] getSortedPaths() { try { // Previously, this got the filesystem of the Table, which could be // different from the filesystem of the partition. FileSystem fs = getDataLocation().getFileSystem(SessionState.getSessionConf()); String pathPattern = getDataLocation().toString(); if (getBucketCount() > 0) { pathPattern = pathPattern + "/*"; } LOG.info("Path pattern = " + pathPattern); FileStatus srcs[] = fs.globStatus(new Path(pathPattern), FileUtils.HIDDEN_FILES_PATH_FILTER); Arrays.sort(srcs); for (FileStatus src : srcs) { LOG.info("Got file: " + src.getPath()); } if (srcs.length == 0) { return null; } return srcs; } catch (Exception e) { throw new RuntimeException("Cannot get path ", e); } }
@Override public boolean exists(final String location) { try { return this.fs.globStatus(new Path(tryHomeDirectory(location) + "*")).length > 0; } catch (final IOException e) { throw new IllegalStateException(e.getMessage(), e); } }