public static boolean runDistCp(List<Path> srcPaths, Path dst, Configuration conf) throws IOException { DistCpOptions options = new DistCpOptions.Builder(srcPaths, dst) .withSyncFolder(true) .withCRC(true) .preserve(FileAttribute.BLOCKSIZE) .build(); // Creates the command-line parameters for distcp List<String> params = constructDistCpParams(srcPaths, dst, conf); try { conf.setBoolean("mapred.mapper.new-api", true); DistCp distcp = new DistCp(conf, options); // HIVE-13704 states that we should use run() instead of execute() due to a hadoop known issue // added by HADOOP-10459 if (distcp.run(params.toArray(new String[params.size()])) == 0) { return true; } else { return false; } } catch (Exception e) { throw new IOException("Cannot execute DistCp process: " + e, e); } finally { conf.setBoolean("mapred.mapper.new-api", false); } }
@Override public boolean runDistCp(List<Path> srcPaths, Path dst, Configuration conf) throws IOException { DistCpOptions options = new DistCpOptions.Builder(srcPaths, dst) .withSyncFolder(true) .withCRC(true) .preserve(FileAttribute.BLOCKSIZE) .build(); // Creates the command-line parameters for distcp List<String> params = constructDistCpParams(srcPaths, dst, conf); try { conf.setBoolean("mapred.mapper.new-api", true); DistCp distcp = new DistCp(conf, options); // HIVE-13704 states that we should use run() instead of execute() due to a hadoop known issue // added by HADOOP-10459 if (distcp.run(params.toArray(new String[0])) == 0) { return true; } else { return false; } } catch (Exception e) { throw new IOException("Cannot execute DistCp process: " + e, e); } finally { conf.setBoolean("mapred.mapper.new-api", false); } }
@Override public void run() { int ret = distCp.run(options.toArray(new String[] {})); retVal.set(Integer.valueOf(ret)); } };
@Override public void run() { int ret = distCp.run(options.toArray(new String[] {})); retVal.set(Integer.valueOf(ret)); } };
@Override public boolean runDistCp(List<Path> srcPaths, Path dst, Configuration conf) throws IOException { DistCpOptions options = new DistCpOptions.Builder(srcPaths, dst) .withSyncFolder(true) .withCRC(true) .preserve(FileAttribute.BLOCKSIZE) .build(); // Creates the command-line parameters for distcp List<String> params = constructDistCpParams(srcPaths, dst, conf); try { conf.setBoolean("mapred.mapper.new-api", true); DistCp distcp = new DistCp(conf, options); // HIVE-13704 states that we should use run() instead of execute() due to a hadoop known issue // added by HADOOP-10459 if (distcp.run(params.toArray(new String[0])) == 0) { return true; } else { return false; } } catch (Exception e) { throw new IOException("Cannot execute DistCp process: " + e, e); } finally { conf.setBoolean("mapred.mapper.new-api", false); } }
assertEquals("Failed to run distcp: " + Arrays.toString(distCpArgs), 0, cp.run(distCpArgs));
/** * test methods run end execute of DistCp class. silple copy file * @throws Exception */ @Test public void testCleanup() throws Exception { Configuration conf = getConf(); Path stagingDir = JobSubmissionFiles.getStagingDir(new Cluster(conf), conf); stagingDir.getFileSystem(conf).mkdirs(stagingDir); Path soure = createFile("tmp.txt"); Path target = createFile("target.txt"); DistCp distcp = new DistCp(conf, null); String[] arg = { soure.toString(), target.toString() }; distcp.run(arg); Assert.assertTrue(fs.exists(target)); }
/** * test methods run end execute of DistCp class. silple copy file * @throws Exception */ @Test public void testCleanup() throws Exception { Configuration conf = getConf(); Path stagingDir = JobSubmissionFiles.getStagingDir(new Cluster(conf), conf); stagingDir.getFileSystem(conf).mkdirs(stagingDir); Path soure = createFile("tmp.txt"); Path target = createFile("target.txt"); DistCp distcp = new DistCp(conf, null); String[] arg = { soure.toString(), target.toString() }; distcp.run(arg); Assert.assertTrue(fs.exists(target)); }