private void copyFile(final Context context, final SnapshotFileInfo inputInfo, final Path outputPath) throws IOException { FileStatus inputStat = getSourceFileStatus(context, inputInfo); if (outputStat != null && sameFile(inputStat, outputStat)) { LOG.info("Skip copy " + inputStat.getPath() + " to " + outputPath + ", same file."); context.getCounter(Counter.FILES_SKIPPED).increment(1); InputStream in = openSourceFile(context, inputInfo); int bandwidthMB = context.getConfiguration().getInt(CONF_BANDWIDTH_MB, 100); if (Integer.MAX_VALUE != bandwidthMB) { createOutputPath(outputPath.getParent()); FSDataOutputStream out = outputFs.create(outputPath, true); try { copyData(context, inputStat.getPath(), in, outputPath, out, inputStat.getLen()); } finally { out.close(); if (!preserveAttributes(outputPath, inputStat)) { LOG.warn("You may have to run manually chown on: " + outputPath); injectTestFailure(context, inputInfo);
/** * Check if the two files are equal by looking at the file length, * and at the checksum (if user has specified the verifyChecksum flag). */ private boolean sameFile(final FileStatus inputStat, final FileStatus outputStat) { // Not matching length if (inputStat.getLen() != outputStat.getLen()) return false; // Mark files as equals, since user asked for no checksum verification if (!verifyChecksum) return true; // If checksums are not available, files are not the same. FileChecksum inChecksum = getFileChecksum(inputFs, inputStat.getPath()); if (inChecksum == null) return false; FileChecksum outChecksum = getFileChecksum(outputFs, outputStat.getPath()); if (outChecksum == null) return false; return inChecksum.equals(outChecksum); } }
private void copyFile(final Context context, final SnapshotFileInfo inputInfo, final Path outputPath) throws IOException { injectTestFailure(context, inputInfo); FileStatus inputStat = getSourceFileStatus(context, inputInfo); if (outputStat != null && sameFile(inputStat, outputStat)) { LOG.info("Skip copy " + inputStat.getPath() + " to " + outputPath + ", same file."); context.getCounter(Counter.FILES_SKIPPED).increment(1); InputStream in = openSourceFile(context, inputInfo); int bandwidthMB = context.getConfiguration().getInt(CONF_BANDWIDTH_MB, 100); if (Integer.MAX_VALUE != bandwidthMB) { createOutputPath(outputPath.getParent()); FSDataOutputStream out = outputFs.create(outputPath, true); try { copyData(context, inputStat.getPath(), in, outputPath, out, inputStat.getLen()); } finally { out.close(); if (!preserveAttributes(outputPath, inputStat)) { LOG.warn("You may have to run manually chown on: " + outputPath);
private void copyFile(final Context context, final SnapshotFileInfo inputInfo, final Path outputPath) throws IOException { FileStatus inputStat = getSourceFileStatus(context, inputInfo); if (outputStat != null && sameFile(inputStat, outputStat)) { LOG.info("Skip copy " + inputStat.getPath() + " to " + outputPath + ", same file."); context.getCounter(Counter.FILES_SKIPPED).increment(1); InputStream in = openSourceFile(context, inputInfo); int bandwidthMB = context.getConfiguration().getInt(CONF_BANDWIDTH_MB, 100); if (Integer.MAX_VALUE != bandwidthMB) { createOutputPath(outputPath.getParent()); FSDataOutputStream out = outputFs.create(outputPath, true); try { copyData(context, inputStat.getPath(), in, outputPath, out, inputStat.getLen()); } finally { out.close(); if (!preserveAttributes(outputPath, inputStat)) { LOG.warn("You may have to run manually chown on: " + outputPath); injectTestFailure(context, inputInfo);
private void copyFile(final Context context, final SnapshotFileInfo inputInfo, final Path outputPath) throws IOException { FileStatus inputStat = getSourceFileStatus(context, inputInfo); if (outputStat != null && sameFile(inputStat, outputStat)) { LOG.info("Skip copy " + inputStat.getPath() + " to " + outputPath + ", same file."); context.getCounter(Counter.FILES_SKIPPED).increment(1); InputStream in = openSourceFile(context, inputInfo); int bandwidthMB = context.getConfiguration().getInt(CONF_BANDWIDTH_MB, 100); if (Integer.MAX_VALUE != bandwidthMB) { createOutputPath(outputPath.getParent()); FSDataOutputStream out = outputFs.create(outputPath, true); try { copyData(context, inputStat.getPath(), in, outputPath, out, inputStat.getLen()); } finally { out.close(); if (!preserveAttributes(outputPath, inputStat)) { LOG.warn("You may have to run manually chown on: " + outputPath); injectTestFailure(context, inputInfo);
private boolean copyFile(final Context context, final Path inputPath, final Path outputPath) throws IOException { FSDataInputStream in = openSourceFile(inputPath); if (in == null) { context.getCounter(Counter.MISSING_FILES).increment(1); FileStatus inputStat = getFileStatus(inputFs, inputPath); if (inputStat == null) return false; FileStatus outputStat = getFileStatus(outputFs, outputPath); if (outputStat != null && sameFile(inputStat, outputStat)) { LOG.info("Skip copy " + inputPath + " to " + outputPath + ", same file."); return true; FSDataOutputStream out = outputFs.create(outputPath, true); try { if (!copyData(context, inputPath, in, outputPath, out, inputStat.getLen())) return false; } finally { return preserveAttributes(outputPath, inputStat); } finally { in.close();
/** * Create the output folder and optionally set ownership. */ private void createOutputPath(final Path path) throws IOException { if (filesUser == null && filesGroup == null) { outputFs.mkdirs(path); } else { Path parent = path.getParent(); if (!outputFs.exists(parent) && !parent.isRoot()) { createOutputPath(parent); } outputFs.mkdirs(path); if (filesUser != null || filesGroup != null) { // override the owner when non-null user/group is specified outputFs.setOwner(path, filesUser, filesGroup); } if (filesMode > 0) { outputFs.setPermission(path, new FsPermission(filesMode)); } } }
/** * Create the output folder and optionally set ownership. */ private void createOutputPath(final Path path) throws IOException { if (filesUser == null && filesGroup == null) { outputFs.mkdirs(path); } else { Path parent = path.getParent(); if (!outputFs.exists(parent) && !parent.isRoot()) { createOutputPath(parent); } outputFs.mkdirs(path); if (filesUser != null || filesGroup != null) { // override the owner when non-null user/group is specified outputFs.setOwner(path, filesUser, filesGroup); } if (filesMode > 0) { outputFs.setPermission(path, new FsPermission(filesMode)); } } }
/** * Create the output folder and optionally set ownership. */ private void createOutputPath(final Path path) throws IOException { if (filesUser == null && filesGroup == null) { outputFs.mkdirs(path); } else { Path parent = path.getParent(); if (!outputFs.exists(parent) && !parent.isRoot()) { createOutputPath(parent); } outputFs.mkdirs(path); if (filesUser != null || filesGroup != null) { // override the owner when non-null user/group is specified outputFs.setOwner(path, filesUser, filesGroup); } if (filesMode > 0) { outputFs.setPermission(path, new FsPermission(filesMode)); } } }
/** * Check if the two files are equal by looking at the file length, * and at the checksum (if user has specified the verifyChecksum flag). */ private boolean sameFile(final FileStatus inputStat, final FileStatus outputStat) { // Not matching length if (inputStat.getLen() != outputStat.getLen()) return false; // Mark files as equals, since user asked for no checksum verification if (!verifyChecksum) return true; // If checksums are not available, files are not the same. FileChecksum inChecksum = getFileChecksum(inputFs, inputStat.getPath()); if (inChecksum == null) return false; FileChecksum outChecksum = getFileChecksum(outputFs, outputStat.getPath()); if (outChecksum == null) return false; return inChecksum.equals(outChecksum); } }
/** * Check if the two files are equal by looking at the file length, * and at the checksum (if user has specified the verifyChecksum flag). */ private boolean sameFile(final FileStatus inputStat, final FileStatus outputStat) { // Not matching length if (inputStat.getLen() != outputStat.getLen()) return false; // Mark files as equals, since user asked for no checksum verification if (!verifyChecksum) return true; // If checksums are not available, files are not the same. FileChecksum inChecksum = getFileChecksum(inputFs, inputStat.getPath()); if (inChecksum == null) return false; FileChecksum outChecksum = getFileChecksum(outputFs, outputStat.getPath()); if (outChecksum == null) return false; return inChecksum.equals(outChecksum); } }
/** * Check if the two files are equal by looking at the file length, * and at the checksum (if user has specified the verifyChecksum flag). */ private boolean sameFile(final FileStatus inputStat, final FileStatus outputStat) { // Not matching length if (inputStat.getLen() != outputStat.getLen()) return false; // Mark files as equals, since user asked for no checksum verification if (!verifyChecksum) return true; // If checksums are not available, files are not the same. FileChecksum inChecksum = getFileChecksum(inputFs, inputStat.getPath()); if (inChecksum == null) return false; FileChecksum outChecksum = getFileChecksum(outputFs, outputStat.getPath()); if (outChecksum == null) return false; return inChecksum.equals(outChecksum); } }
/** * Check if the two files are equal by looking at the file length, * and at the checksum (if user has specified the verifyChecksum flag). */ private boolean sameFile(final FileStatus inputStat, final FileStatus outputStat) { // Not matching length if (inputStat.getLen() != outputStat.getLen()) return false; // Mark files as equals, since user asked for no checksum verification if (!verifyChecksum) return true; // If checksums are not available, files are not the same. FileChecksum inChecksum = getFileChecksum(inputFs, inputStat.getPath()); if (inChecksum == null) return false; FileChecksum outChecksum = getFileChecksum(outputFs, outputStat.getPath()); if (outChecksum == null) return false; return inChecksum.equals(outChecksum); }
@Override public void map(Text key, NullWritable value, Context context) throws InterruptedException, IOException { Path inputPath = new Path(key.toString()); Path outputPath = getOutputPath(inputPath); LOG.info("copy file input=" + inputPath + " output=" + outputPath); if (copyFile(context, inputPath, outputPath)) { LOG.info("copy completed for input=" + inputPath + " output=" + outputPath); } }
@Override public void map(BytesWritable key, NullWritable value, Context context) throws InterruptedException, IOException { SnapshotFileInfo inputInfo = SnapshotFileInfo.parseFrom(key.copyBytes()); Path outputPath = getOutputPath(inputInfo); copyFile(context, inputInfo, outputPath); }
@Override public void map(BytesWritable key, NullWritable value, Context context) throws InterruptedException, IOException { SnapshotFileInfo inputInfo = SnapshotFileInfo.parseFrom(key.copyBytes()); Path outputPath = getOutputPath(inputInfo); copyFile(context, inputInfo, outputPath); }
@Override public void map(BytesWritable key, NullWritable value, Context context) throws InterruptedException, IOException { SnapshotFileInfo inputInfo = SnapshotFileInfo.parseFrom(key.copyBytes()); Path outputPath = getOutputPath(inputInfo); copyFile(context, inputInfo, outputPath); }
/** * Create the output folder and optionally set ownership. */ private void createOutputPath(final Path path) throws IOException { if (filesUser == null && filesGroup == null) { outputFs.mkdirs(path); } else { Path parent = path.getParent(); if (!outputFs.exists(parent) && !parent.isRoot()) { createOutputPath(parent); } outputFs.mkdirs(path); if (filesUser != null || filesGroup != null) { // override the owner when non-null user/group is specified outputFs.setOwner(path, filesUser, filesGroup); } if (filesMode > 0) { outputFs.setPermission(path, new FsPermission(filesMode)); } } }
@Override public void map(BytesWritable key, NullWritable value, Context context) throws InterruptedException, IOException { SnapshotFileInfo inputInfo = SnapshotFileInfo.parseFrom(key.copyBytes()); Path outputPath = getOutputPath(inputInfo); copyFile(context, inputInfo, outputPath); }