public Path getTargetPath(Path publishDir, Path originPath, String identifier) { Path filePathRelativeToSearchPath = PathUtils.relativizePath(originPath, this.rootPath); return new Path(publishDir, filePathRelativeToSearchPath); }
private Map<Path, FileStatus> createPathMap(List<FileStatus> files, Path prefix) { Map<Path, FileStatus> map = Maps.newHashMap(); for (FileStatus status : files) { map.put(PathUtils.relativizePath(status.getPath(), prefix), status); } return map; }
public URI computeURI(Path filePath) { // Make sure this is relative URI uri = PathUtils.relativizePath(filePath, jobConfigDirPath).toUri(); if (this.extensionToStrip.isPresent()) { uri = PathUtils.removeExtension(new Path(uri), this.extensionToStrip.get()).toUri(); } return uri; }
protected static Path replacedPrefix(Path sourcePath, Path prefixTobeReplaced, Path prefixReplacement) { Path sourcePathWithoutSchemeAndAuthority = PathUtils.getPathWithoutSchemeAndAuthority(sourcePath); Preconditions.checkArgument(PathUtils.isAncestor(prefixTobeReplaced, sourcePathWithoutSchemeAndAuthority), "When replacing prefix, all locations must be descendants of the prefix. " + "The prefix: %s, file location: %s.", prefixTobeReplaced, sourcePathWithoutSchemeAndAuthority); Path relativePath = PathUtils.relativizePath(sourcePathWithoutSchemeAndAuthority, prefixTobeReplaced); Path result = new Path(prefixReplacement, relativePath); return result; }
@Override public void run() { for (Path locationToCopy : locationsToCopy) { long timestampFromPath = this.versionDatetime.getMillis(); try { for (FileStatus singleFile : this.srcFs.listStatus(locationToCopy, this.filter)) { Path singleFilePath = singleFile.getPath(); log.debug("Checking if it is a copyable file: " + singleFilePath); Path relativePath = PathUtils.relativizePath(singleFilePath, datasetRoot); Path targetPath = new Path(targetRoot, relativePath); if (this.isCopyableFile(singleFile, targetPath)) { log.debug("Will create workunit for: " + singleFilePath); copyableFileList .add(this.generateCopyableFile(singleFile, targetPath, timestampFromPath, locationToCopy)); } } } catch (IOException e) { e.printStackTrace(); throw new RuntimeException("Failed to get copyable files for " + locationToCopy, e); } } }
/** * Checks whether possibleAncestor is an ancestor of fullPath. * @param possibleAncestor Possible ancestor of fullPath. * @param fullPath path to check. * @return true if possibleAncestor is an ancestor of fullPath. */ public static boolean isAncestor(Path possibleAncestor, Path fullPath) { return !relativizePath(fullPath, possibleAncestor).equals(getPathWithoutSchemeAndAuthority(fullPath)); }
Path pathRelativeToTrash = PathUtils.relativizePath(fileStatus.getPath(), this.trashLocation); Path targetPath = new Path(snapshotDir, pathRelativeToTrash); boolean movedThisPath = true;
/** * Find dataset versions in the input {@link org.apache.hadoop.fs.Path}. Dataset versions are subdirectories of the * input {@link org.apache.hadoop.fs.Path} representing a single manageable unit in the dataset. * See {@link gobblin.data.management.retention.DatasetCleaner} for more information. * * @param dataset {@link org.apache.hadoop.fs.Path} to directory containing all versions of a dataset. * @return Map of {@link gobblin.data.management.version.DatasetVersion} and {@link org.apache.hadoop.fs.FileStatus} * for each dataset version found. * @throws IOException */ @Override public Collection<T> findDatasetVersions(Dataset dataset) throws IOException { FileSystemDataset fsDataset = (FileSystemDataset) dataset; Path versionGlobStatus = new Path(fsDataset.datasetRoot(), globVersionPattern()); FileStatus[] dataSetVersionPaths = this.fs.globStatus(versionGlobStatus); List<T> dataSetVersions = Lists.newArrayList(); for (FileStatus dataSetVersionPath : dataSetVersionPaths) { T datasetVersion = getDatasetVersion(PathUtils.relativizePath(dataSetVersionPath.getPath(), fsDataset.datasetRoot()), dataSetVersionPath); if (datasetVersion != null) { dataSetVersions.add(datasetVersion); } } return dataSetVersions; }
/** * Get a {@link CopyableFile.Builder}. * * @param originFs {@link FileSystem} where original file exists. * @param origin {@link FileStatus} of the original file. * @param datasetRoot Value of {@link CopyableDataset#datasetRoot} of the dataset creating this {@link CopyableFile}. * @param copyConfiguration {@link CopyConfiguration} for the copy job. * @return a {@link CopyableFile.Builder}. * @deprecated use {@link #fromOriginAndDestination}. This method was changed to remove reliance on dataset root * which is not standard of all datasets. The old functionality on inferring destinations cannot be * achieved without dataset root and common dataset root, so this is an approximation. Copyable datasets * should compute file destinations themselves. */ @Deprecated public static Builder builder(FileSystem originFs, FileStatus origin, Path datasetRoot, CopyConfiguration copyConfiguration) { Path relativePath = PathUtils.relativizePath(origin.getPath(), datasetRoot); Path targetRoot = new Path(copyConfiguration.getPublishDir(), datasetRoot.getName()); Path targetPath = new Path(targetRoot, relativePath); return _hiddenBuilder().originFS(originFs).origin(origin).destination(targetPath) .preserve(copyConfiguration.getPreserve()).configuration(copyConfiguration); }
+ "Table root location: %s, file location: %s.", this.dataset.getTableRootPath(), sourcePath); Path relativePath = PathUtils.relativizePath(sourcePath, this.dataset.getTableRootPath().get()); return targetFs.makeQualified(new Path(this.targetTableRoot.get(), relativePath)); } else {
Path targetPath = new Path(configuration.getPublishDir(), PathUtils.relativizePath(this.rootPath, nonGlobSearchPath)); Path filePathRelativeToSearchPath = PathUtils.relativizePath(file.getPath(), nonGlobSearchPath); Path thisTargetPath = new Path(configuration.getPublishDir(), filePathRelativeToSearchPath);
Path relative = PathUtils.relativizePath(PathUtils.getPathWithoutSchemeAndAuthority(originFileStatus.getPath()), PathUtils.getPathWithoutSchemeAndAuthority(copyFrom.getDatasetPath()));