@SuppressWarnings("unchecked") private List<Path> getSourcePaths(Field fieldInputOptions) throws IOException{ Object options; try { options = fieldInputOptions.get(this); if (options instanceof DistCpOptions) { return ((DistCpOptions) options).getSourcePaths(); } else { // Hadoop 3 Class<?> classContext = Class.forName("org.apache.hadoop.tools.DistCpContext"); Method methodGetSourcePaths = classContext.getDeclaredMethod("getSourcePaths"); methodGetSourcePaths.setAccessible(true); return (List<Path>) methodGetSourcePaths.invoke(options); } } catch (IllegalArgumentException | IllegalAccessException | ClassNotFoundException | NoSuchMethodException | SecurityException | InvocationTargetException e) { throw new IOException(e); } }
public DistCpContext(DistCpOptions options) { this.options = options; this.sourcePaths = options.getSourcePaths(); }
private Path computeSourceRootPath(FileStatus sourceStatus, DistCpOptions options) throws IOException { Path target = options.getTargetPath(); FileSystem targetFS = target.getFileSystem(getConf()); final boolean targetPathExists = options.getTargetPathExists(); boolean solitaryFile = options.getSourcePaths().size() == 1 && !sourceStatus.isDirectory(); if (solitaryFile) { if (targetFS.isFile(target) || !targetPathExists) { return sourceStatus.getPath(); } else { return sourceStatus.getPath().getParent(); } } else { boolean specialHandling = (options.getSourcePaths().size() == 1 && !targetPathExists) || options.shouldSyncFolder() || options.shouldOverwrite(); return specialHandling && sourceStatus.isDirectory() ? sourceStatus.getPath() : sourceStatus.getPath().getParent(); } }
/** * Implementation of CopyListing::buildListing(). * Creates the copy listing by "globbing" all source-paths. * @param pathToListingFile The location at which the copy-listing file * is to be created. * @param options Input Options for DistCp (indicating source/target paths.) * @throws IOException */ @Override public void doBuildListing(Path pathToListingFile, DistCpOptions options) throws IOException { List<Path> globbedPaths = new ArrayList<Path>(); if (options.getSourcePaths().isEmpty()) { throw new InvalidInputException("Nothing to process. Source paths::EMPTY"); } for (Path p : options.getSourcePaths()) { FileSystem fs = p.getFileSystem(getConf()); FileStatus[] inputs = fs.globStatus(p); if(inputs != null && inputs.length > 0) { for (FileStatus onePath: inputs) { globbedPaths.add(onePath.getPath()); } } else { throw new InvalidInputException(p + " doesn't exist"); } } DistCpOptions optionsGlobbed = new DistCpOptions(options); optionsGlobbed.setSourcePaths(globbedPaths); simpleListing.buildListing(pathToListingFile, optionsGlobbed); }
@Override public void doBuildListing(Path pathToListFile, DistCpOptions options) throws IOException { try (Writer writer = newWriter(pathToListFile)) { Path sourceRootPath = getRootPath(getConf()); for (Path sourcePath : options.getSourcePaths()) { FileSystem fileSystem = sourcePath.getFileSystem(getConf()); FileStatus directory = fileSystem.getFileStatus(sourcePath); Map<String, CopyListingFileStatus> children = new FileStatusTreeTraverser(fileSystem) .preOrderTraversal(directory) .transform(new CopyListingFileStatusFunction(fileSystem, options)) .uniqueIndex(new RelativePathFunction(sourceRootPath)); for (Entry<String, CopyListingFileStatus> entry : children.entrySet()) { LOG.debug("Adding '{}' with relative path '{}'", entry.getValue().getPath(), entry.getKey()); writer.append(new Text(entry.getKey()), entry.getValue()); writer.sync(); } } } }
@Override public void doBuildListing(Path pathToListFile, DistCpOptions options) throws IOException { try (Writer writer = newWriter(pathToListFile)) { Path sourceRootPath = getRootPath(getConf()); for (Path sourcePath : options.getSourcePaths()) { FileSystem fileSystem = sourcePath.getFileSystem(getConf()); FileStatus directory = fileSystem.getFileStatus(sourcePath); Map<String, CopyListingFileStatus> children = new FileStatusTreeTraverser(fileSystem) .preOrderTraversal(directory) .transform(new CopyListingFileStatusFunction(fileSystem, options)) .uniqueIndex(new RelativePathFunction(sourceRootPath)); for (Entry<String, CopyListingFileStatus> entry : children.entrySet()) { LOG.debug("Adding '{}' with relative path '{}'", entry.getValue().getPath(), entry.getKey()); writer.append(new Text(entry.getKey()), entry.getValue()); writer.sync(); } } } }
if (options.getSourcePaths().size() > 1) { throw new InvalidInputException("Multiple source being copied to a file: " + targetPath); Path srcPath = options.getSourcePaths().get(0); FileSystem sourceFS = srcPath.getFileSystem(getConf()); if (!sourceFS.isFile(srcPath)) { for (Path path: options.getSourcePaths()) { FileSystem fs = path.getFileSystem(getConf()); if (!fs.exists(path)) { Path[] inputPaths = options.getSourcePaths().toArray(new Path[1]); TokenCache.obtainTokensForNamenodes(credentials, inputPaths, getConf());
static boolean sync(DistCpOptions inputOptions, Configuration conf) throws IOException { List<Path> sourcePaths = inputOptions.getSourcePaths(); if (sourcePaths.size() != 1) {
DistCpOptions options) throws IOException { try { for (Path path: options.getSourcePaths()) { FileSystem sourceFS = path.getFileSystem(getConf()); final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL);
/** * Copy constructor. * @param that DistCpOptions being copied from. */ public DistCpOptions(DistCpOptions that) { if (this != that && that != null) { this.atomicCommit = that.atomicCommit; this.syncFolder = that.syncFolder; this.deleteMissing = that.deleteMissing; this.ignoreFailures = that.ignoreFailures; this.overwrite = that.overwrite; this.skipCRC = that.skipCRC; this.blocking = that.blocking; this.maxMaps = that.maxMaps; this.mapBandwidth = that.mapBandwidth; this.sslConfigurationFile = that.getSslConfigurationFile(); this.copyStrategy = that.copyStrategy; this.preserveStatus = that.preserveStatus; this.preserveRawXattrs = that.preserveRawXattrs; this.atomicWorkPath = that.getAtomicWorkPath(); this.logPath = that.getLogPath(); this.sourceFileListing = that.getSourceFileListing(); this.sourcePaths = that.getSourcePaths(); this.targetPath = that.getTargetPath(); this.targetPathExists = that.getTargetPathExists(); } }