totalBytesExpected += fst.getLen(); Text key = getKey(path); writer.append(key, new CopyListingFileStatus(fst));
private void compareFileLengths(CopyListingFileStatus source, Path target, Configuration configuration, long targetLen) throws IOException { final Path sourcePath = source.getPath(); FileSystem fs = sourcePath.getFileSystem(configuration); long srcLen = fs.getFileStatus(sourcePath).getLen(); if (srcLen != targetLen) throw new IOException("Mismatch in length of source:" + sourcePath + " (" + srcLen + ") and target:" + target + " (" + targetLen + ")"); }
private String getFileType(CopyListingFileStatus fileStatus) { if (null == fileStatus) { return "N/A"; } return fileStatus.isDirectory() ? "dir" : "file"; }
public boolean isSplit() { return getChunkLength() != Long.MAX_VALUE && getChunkLength() != getLen(); }
public long getSizeToCopy() { return isSplit()? getChunkLength() : getLen(); }
@Test public void testPreserveTimestampOnDirectory() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.TIMES); Path dst = new Path("/tmp/abc"); Path src = new Path("/tmp/src"); createDirectory(fs, src); createDirectory(fs, dst); fs.setPermission(src, fullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setTimes(src, 0, 0); fs.setPermission(dst, noPerm); fs.setOwner(dst, "nobody", "nobody-group"); fs.setTimes(dst, 100, 100); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); // FileStatus.equals only compares path field, must explicitly compare all fields Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime()); Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime()); }
@Override public void write(DataOutput out) throws IOException { Text.writeString(out, getPath().toString(), Text.DEFAULT_MAX_LEN); out.writeLong(getLen()); out.writeBoolean(isDirectory()); out.writeShort(getReplication()); out.writeLong(getBlockSize()); out.writeLong(getModificationTime()); out.writeLong(getAccessTime()); out.writeShort(getPermission().toShort()); Text.writeString(out, getOwner(), Text.DEFAULT_MAX_LEN); Text.writeString(out, getGroup(), Text.DEFAULT_MAX_LEN); if (aclEntries != null) {
@Test public void testPreserveGroupOnDirectory() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.GROUP); Path dst = new Path("/tmp/abc"); Path src = new Path("/tmp/src"); createDirectory(fs, src); createDirectory(fs, dst); fs.setPermission(src, fullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setPermission(dst, noPerm); fs.setOwner(dst, "nobody", "nobody-group"); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); // FileStatus.equals only compares path field, must explicitly compare all fields Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup())); }
List<AclEntry> srcAcl = srcFileStatus.getAclEntries(); List<AclEntry> targetAcl = getAcl(targetFS, targetFileStatus); if (!srcAcl.equals(targetAcl)) { if (srcFileStatus.getPermission().getStickyBit() != targetFileStatus.getPermission().getStickyBit()) { targetFS.setPermission(path, srcFileStatus.getPermission()); !srcFileStatus.getPermission().equals(targetFileStatus.getPermission())) { targetFS.setPermission(path, srcFileStatus.getPermission()); final String rawNS = StringUtils.toLowerCase(XAttr.NameSpace.RAW.name()); Map<String, byte[]> srcXAttrs = srcFileStatus.getXAttrs(); Map<String, byte[]> targetXAttrs = getXAttrs(targetFS, path); if (srcXAttrs != null && !srcXAttrs.equals(targetXAttrs)) { !targetFileStatus.isDirectory() && !targetFileStatus.isErasureCoded() && !srcFileStatus.isErasureCoded() && srcFileStatus.getReplication() != targetFileStatus.getReplication()) { targetFS.setReplication(path, srcFileStatus.getReplication()); !group.equals(srcFileStatus.getGroup())) { group = srcFileStatus.getGroup(); chown = true; !user.equals(srcFileStatus.getOwner())) { user = srcFileStatus.getOwner();
private void verifyContents(Path listingPath) throws Exception { SequenceFile.Reader reader = new SequenceFile.Reader(cluster.getFileSystem(), listingPath, new Configuration()); Text key = new Text(); CopyListingFileStatus value = new CopyListingFileStatus(); Map<String, String> actualValues = new HashMap<String, String>(); while (reader.next(key, value)) { if (value.isDirectory() && key.toString().equals("")) { // ignore root with empty relPath, which is an entry to be // used for preserving root attributes etc. continue; } actualValues.put(value.getPath().toString(), key.toString()); } Assert.assertEquals(expectedValues.size(), actualValues.size()); for (Map.Entry<String, String> entry : actualValues.entrySet()) { Assert.assertEquals(entry.getValue(), expectedValues.get(entry.getKey())); } } }
@Override public String toString() { StringBuilder sb = new StringBuilder(super.toString()); sb.append('{'); sb.append(this.getPath().toString()); sb.append(" length = ").append(this.getLen()); sb.append(" aclEntries = ").append(aclEntries); sb.append(", xAttrs = ").append(xAttrs); if (isSplit()) { sb.append(", chunkOffset = ").append(this.getChunkOffset()); sb.append(", chunkLength = ").append(this.getChunkLength()); } sb.append('}'); return sb.toString(); } }
CopyListingFileStatus srcFileStatus = new CopyListingFileStatus(); Text srcRelPath = new Text(); CopyListingFileStatus lastFileStatus = null; if (srcFileStatus.isDirectory()) { continue; if (srcFileStatus.getChunkOffset() + srcFileStatus.getChunkLength() == srcFileStatus.getLen()) { } else { if (lastFileStatus == null) { lastFileStatus = new CopyListingFileStatus(srcFileStatus); } else { if (!srcFileStatus.getPath().equals(lastFileStatus.getPath()) || srcFileStatus.getChunkOffset() != (lastFileStatus.getChunkOffset() + lastFileStatus.getChunkLength())) { String emsg = "Inconsistent sequence file: current " + "chunk file " + srcFileStatus + " doesnt match prior " + lastFileStatus.setChunkOffset(srcFileStatus.getChunkOffset()); lastFileStatus.setChunkLength(srcFileStatus.getChunkLength());
try { Text lastKey = new Text("*"); //source relative path can never hold * CopyListingFileStatus lastFileStatus = new CopyListingFileStatus(); while (reader.next(currentKey)) { if (currentKey.equals(lastKey)) { CopyListingFileStatus currentFileStatus = new CopyListingFileStatus(); reader.getCurrentValue(currentFileStatus); throw new DuplicateFileException("File " + lastFileStatus.getPath() + " and " + currentFileStatus.getPath() + " would cause duplicates. Aborting"); FileSystem lastFs = lastFileStatus.getPath().getFileSystem(config); URI lastFsUri = lastFs.getUri(); if (!aclSupportCheckFsSet.contains(lastFsUri)) { FileSystem lastFs = lastFileStatus.getPath().getFileSystem(config); URI lastFsUri = lastFs.getUri(); if (!xAttrSupportCheckFsSet.contains(lastFsUri)) {
private void writeToFileListingRoot(SequenceFile.Writer fileListWriter, CopyListingFileStatus fileStatus, Path sourcePathRoot, DistCpOptions options) throws IOException { boolean syncOrOverwrite = options.shouldSyncFolder() || options.shouldOverwrite(); if (fileStatus.getPath().equals(sourcePathRoot) && fileStatus.isDirectory() && syncOrOverwrite) { // Skip the root-paths when syncOrOverwrite if (LOG.isDebugEnabled()) { LOG.debug("Skip " + fileStatus.getPath()); } return; } writeToFileListing(fileListWriter, fileStatus, sourcePathRoot, options); }
private void writeToFileListing(SequenceFile.Writer fileListWriter, CopyListingFileStatus fileStatus, Path sourcePathRoot, DistCpOptions options) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("REL PATH: " + DistCpUtils.getRelativePath(sourcePathRoot, fileStatus.getPath()) + ", FULL PATH: " + fileStatus.getPath()); } FileStatus status = fileStatus; if (!shouldCopy(fileStatus.getPath(), options)) { return; } fileListWriter.append(new Text(DistCpUtils.getRelativePath(sourcePathRoot, fileStatus.getPath())), status); fileListWriter.sync(); if (!fileStatus.isDirectory()) { totalBytesToCopy += fileStatus.getLen(); } totalPaths++; } }
long lastChunkOffset = -1; long lastChunkLength = -1; CopyListingFileStatus lastFileStatus = new CopyListingFileStatus(); while (reader.next(currentKey)) { if (currentKey.equals(lastKey)) { CopyListingFileStatus currentFileStatus = new CopyListingFileStatus(); reader.getCurrentValue(currentFileStatus); if (!splitLargeFile) { throw new DuplicateFileException("File " + lastFileStatus.getPath() + " and " + currentFileStatus.getPath() + " would cause duplicates. Aborting"); } else { if (lastChunkOffset + lastChunkLength != currentFileStatus.getChunkOffset()) { throw new InvalidInputException("File " + lastFileStatus.getPath() + " " + lastChunkOffset + "," + lastChunkLength + " and " + currentFileStatus.getPath() + " " + currentFileStatus.getChunkOffset() + "," + currentFileStatus.getChunkLength() + " are not continuous. Aborting"); FileSystem lastFs = lastFileStatus.getPath().getFileSystem(config); URI lastFsUri = lastFs.getUri(); if (!aclSupportCheckFsSet.contains(lastFsUri)) { FileSystem lastFs = lastFileStatus.getPath().getFileSystem(config); URI lastFsUri = lastFs.getUri(); if (!xAttrSupportCheckFsSet.contains(lastFsUri)) {
CopyListingFileStatus srcFileStatus = new CopyListingFileStatus(); Text srcRelPath = new Text(); if (! srcFileStatus.isDirectory()) continue;
private boolean canSkip(FileSystem sourceFS, CopyListingFileStatus source, FileStatus target) throws IOException { if (!syncFolders) { return true; } boolean sameLength = target.getLen() == source.getLen(); boolean sameBlockSize = source.getBlockSize() == target.getBlockSize() || !preserve.contains(FileAttribute.BLOCKSIZE); if (sameLength && sameBlockSize) { return skipCrc || DistCpUtils.checksumsAreEqual(sourceFS, source.getPath(), null, targetFS, target.getPath()); } else { return false; } }
private void writeToFileListing(SequenceFile.Writer fileListWriter, CopyListingFileStatus fileStatus, Path sourcePathRoot) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("REL PATH: " + DistCpUtils.getRelativePath(sourcePathRoot, fileStatus.getPath()) + ", FULL PATH: " + fileStatus.getPath()); } if (!shouldCopy(fileStatus.getPath())) { return; } fileListWriter.append(new Text(DistCpUtils.getRelativePath(sourcePathRoot, fileStatus.getPath())), fileStatus); fileListWriter.sync(); if (!fileStatus.isDirectory()) { totalBytesToCopy += fileStatus.getSizeToCopy(); } else { totalDirs++; } totalPaths++; maybePrintStats(); } }
private void copyFileWithRetry(String description, CopyListingFileStatus sourceFileStatus, Path target, FileStatus targrtFileStatus, Context context, FileAction action, EnumSet<DistCpOptions.FileAttribute> fileAttributes) throws IOException, InterruptedException { long bytesCopied; try { bytesCopied = (Long) new RetriableFileCopyCommand(skipCrc, description, action).execute(sourceFileStatus, target, context, fileAttributes); } catch (Exception e) { context.setStatus("Copy Failure: " + sourceFileStatus.getPath()); throw new IOException("File copy failed: " + sourceFileStatus.getPath() + " --> " + target, e); } incrementCounter(context, Counter.BYTESEXPECTED, sourceFileStatus.getLen()); incrementCounter(context, Counter.BYTESCOPIED, bytesCopied); incrementCounter(context, Counter.COPY, 1); totalBytesCopied += bytesCopied; if (verboseLog) { context.write(null, new Text("FILE_COPIED: source=" + sourceFileStatus.getPath() + "," + " size=" + sourceFileStatus.getLen() + " --> " + "target=" + target + ", size=" + (targrtFileStatus == null ? 0 : targrtFileStatus.getLen()))); } }