protected static Path replacedPrefix(Path sourcePath, Path prefixTobeReplaced, Path prefixReplacement) { Path sourcePathWithoutSchemeAndAuthority = PathUtils.getPathWithoutSchemeAndAuthority(sourcePath); Preconditions.checkArgument(PathUtils.isAncestor(prefixTobeReplaced, sourcePathWithoutSchemeAndAuthority), "When replacing prefix, all locations must be descendants of the prefix. " + "The prefix: %s, file location: %s.", prefixTobeReplaced, sourcePathWithoutSchemeAndAuthority); Path relativePath = PathUtils.relativizePath(sourcePathWithoutSchemeAndAuthority, prefixTobeReplaced); Path result = new Path(prefixReplacement, relativePath); return result; }
/** * * @param jobConfDirPath The directory path for job cofniguration files. * @param uri Uri as the identifier of JobSpec * @return */ protected Path getPathForURI(Path jobConfDirPath, URI uri) { return PathUtils.mergePaths(jobConfDirPath, new Path(uri)); }
/** * * @param fsSpecStoreDirPath The directory path for specs. * @param uri Uri as the identifier of JobSpec * @return */ protected Path getPathForURI(Path fsSpecStoreDirPath, URI uri, String version) { return PathUtils.addExtension(PathUtils.mergePaths(fsSpecStoreDirPath, new Path(uri)), version); } }
/** * Checks whether possibleAncestor is an ancestor of fullPath. * @param possibleAncestor Possible ancestor of fullPath. * @param fullPath path to check. * @return true if possibleAncestor is an ancestor of fullPath. */ public static boolean isAncestor(Path possibleAncestor, Path fullPath) { return !relativizePath(fullPath, possibleAncestor).equals(getPathWithoutSchemeAndAuthority(fullPath)); }
public SubsetFilesCopyableDataset(final FileSystem fs, Path rootPath, Properties properties, String idenifier, List<FileStatus> subFiles) { this.rootPath = PathUtils.getPathWithoutSchemeAndAuthority(rootPath); this.fs = fs; this.files = subFiles; this.identifier = idenifier; this.props = properties; }
/** * Deletes empty directories starting with startPath and all ancestors up to but not including limitPath. * @param fs {@link FileSystem} where paths are located. * @param limitPath only {@link Path}s that are strict descendants of this path will be deleted. * @param startPath first {@link Path} to delete. Afterwards empty ancestors will be deleted. * @throws IOException */ public static void deleteEmptyParentDirectories(FileSystem fs, Path limitPath, Path startPath) throws IOException { if (PathUtils.isAncestor(limitPath, startPath) && !PathUtils.getPathWithoutSchemeAndAuthority(limitPath) .equals(PathUtils.getPathWithoutSchemeAndAuthority(startPath)) && fs.listStatus(startPath).length == 0) { if (!fs.delete(startPath, false)) { log.warn("Failed to delete empty directory " + startPath); } else { log.info("Deleted empty directory " + startPath); } deleteEmptyParentDirectories(fs, limitPath, startPath.getParent()); } }
public Path getTargetPath(Path publishDir, Path originPath, String identifier) { Path filePathRelativeToSearchPath = PathUtils.relativizePath(originPath, this.rootPath); return new Path(publishDir, filePathRelativeToSearchPath); }
public URI computeURI(Path filePath) { // Make sure this is relative URI uri = PathUtils.relativizePath(filePath, jobConfigDirPath).toUri(); if (this.extensionToStrip.isPresent()) { uri = PathUtils.removeExtension(new Path(uri), this.extensionToStrip.get()).toUri(); } return uri; }
public static Path getOutputFilePath(CopyableFile file, Path outputDir, CopyEntity.DatasetAndPartition datasetAndPartition) { Path destinationWithoutSchemeAndAuthority = PathUtils.getPathWithoutSchemeAndAuthority(file.getDestination()); return new Path(getPartitionOutputRoot(outputDir, datasetAndPartition), PathUtils.withoutLeadingSeparator(destinationWithoutSchemeAndAuthority)); }
/** * Finds the deepest ancestor of input that is not a glob. */ public static Path deepestNonGlobPath(Path input) { Path commonRoot = input; while (commonRoot != null && isGlob(commonRoot)) { commonRoot = commonRoot.getParent(); } return commonRoot; }
throws IOException { Path nonGlobSearchPath = PathUtils.deepestNonGlobPath(this.glob); Path targetPath = new Path(configuration.getPublishDir(), PathUtils.relativizePath(this.rootPath, nonGlobSearchPath)); Path filePathRelativeToSearchPath = PathUtils.relativizePath(file.getPath(), nonGlobSearchPath); Path thisTargetPath = new Path(configuration.getPublishDir(), filePathRelativeToSearchPath);
Path newPath = PathUtils.removeExtension(pullFile.getPath(), PullFileLoader.DEFAULT_JAVA_PROPS_PULL_FILE_EXTENSIONS.toArray(new String[]{})); newPath = PathUtils.addExtension(newPath, "conf"); newPath = new Path(this.outputPath, newPath.getName());
/** * Compute the correct {@link OwnerAndPermission} obtained from replicating source owner and permissions and applying * the {@link PreserveAttributes} rules for fromPath and every ancestor up to but excluding toPath. * * @return A list of the computed {@link OwnerAndPermission}s starting from fromPath, up to but excluding toPath. * @throws IOException if toPath is not an ancestor of fromPath. */ public static List<OwnerAndPermission> resolveReplicatedOwnerAndPermissionsRecursively(FileSystem sourceFs, Path fromPath, Path toPath, CopyConfiguration copyConfiguration) throws IOException { if (!PathUtils.isAncestor(toPath, fromPath)) { throw new IOException(String.format("toPath %s must be an ancestor of fromPath %s.", toPath, fromPath)); } List<OwnerAndPermission> ownerAndPermissions = Lists.newArrayList(); Path currentPath = fromPath; while (PathUtils.isAncestor(toPath, currentPath.getParent())) { ownerAndPermissions.add(resolveReplicatedOwnerAndPermission(sourceFs, currentPath, copyConfiguration)); currentPath = currentPath.getParent(); } return ownerAndPermissions; }
Path readyFilePath = PathUtils.addExtension(cf.getOrigin().getPath(), READY_EXTENSION); try { if (sourceFs.exists(readyFilePath)) {
/** * Deletes any empty paths in <code>possiblyEmptyDirectories</code> all the way upto the {@link FileSystemDataset#datasetRoot()}. */ public void cleanEmptyDirectories(final Set<Path> possiblyEmptyDirectories, final FileSystemDataset fsDataset) throws IOException { if (this.deleteEmptyDirectories && !this.simulate) { for (Path parentDirectory : possiblyEmptyDirectories) { PathUtils.deleteEmptyParentDirectories(fs, fsDataset.datasetRoot(), parentDirectory); } } } }
private void modifyExtensionAtDestination(CopyableFile file) { if (extensionsToRemove().size() > 0) { file.setDestination(PathUtils.removeExtension(file.getDestination(), extensionsToRemove().toArray(new String[0]))); } } }
@Override public Path call() throws IOException { Path versionRootPath = PathUtils.combinePaths(SimpleHadoopFilesystemConfigStore.this.physicalStoreRoot.toString(), CONFIG_STORE_NAME, this.version); if (SimpleHadoopFilesystemConfigStore.this.fs.isDirectory(versionRootPath)) { return versionRootPath; } throw new VersionDoesNotExistException(getStoreURI(), this.version, String.format("Cannot find specified version under root %s", versionRootPath)); } }
/** * Compare two path without shedme and authority (the prefix) * @param path1 * @param path2 * @return */ public static boolean compareWithoutSchemeAndAuthority(Path path1, Path path2) { return PathUtils.getPathWithoutSchemeAndAuthority(path1).equals(getPathWithoutSchemeAndAuthority(path2)); } }
for(FileStatus f: allFilesInTarget){ if(pathFilter.accept(f.getPath())){ copyToFileMap.put(PathUtils.getPathWithoutSchemeAndAuthority(f.getPath()), f); Path relative = PathUtils.relativizePath(PathUtils.getPathWithoutSchemeAndAuthority(originFileStatus.getPath()), PathUtils.getPathWithoutSchemeAndAuthority(copyFrom.getDatasetPath())); .fileSet(PathUtils.getPathWithoutSchemeAndAuthority(copyTo.getDatasetPath()).toString()).build());
private Map<Path, FileStatus> createPathMap(List<FileStatus> files, Path prefix) { Map<Path, FileStatus> map = Maps.newHashMap(); for (FileStatus status : files) { map.put(PathUtils.relativizePath(status.getPath(), prefix), status); } return map; }