public static boolean runDistCp(List<Path> srcPaths, Path dst, Configuration conf) throws IOException { DistCpOptions options = new DistCpOptions.Builder(srcPaths, dst) .withSyncFolder(true) .withCRC(true) .preserve(FileAttribute.BLOCKSIZE) .build(); // Creates the command-line parameters for distcp List<String> params = constructDistCpParams(srcPaths, dst, conf); try { conf.setBoolean("mapred.mapper.new-api", true); DistCp distcp = new DistCp(conf, options); // HIVE-13704 states that we should use run() instead of execute() due to a hadoop known issue // added by HADOOP-10459 if (distcp.run(params.toArray(new String[params.size()])) == 0) { return true; } else { return false; } } catch (Exception e) { throw new IOException("Cannot execute DistCp process: " + e, e); } finally { conf.setBoolean("mapred.mapper.new-api", false); } }
@Override public boolean runDistCp(List<Path> srcPaths, Path dst, Configuration conf) throws IOException { DistCpOptions options = new DistCpOptions.Builder(srcPaths, dst) .withSyncFolder(true) .withCRC(true) .preserve(FileAttribute.BLOCKSIZE) .build(); // Creates the command-line parameters for distcp List<String> params = constructDistCpParams(srcPaths, dst, conf); try { conf.setBoolean("mapred.mapper.new-api", true); DistCp distcp = new DistCp(conf, options); // HIVE-13704 states that we should use run() instead of execute() due to a hadoop known issue // added by HADOOP-10459 if (distcp.run(params.toArray(new String[0])) == 0) { return true; } else { return false; } } catch (Exception e) { throw new IOException("Cannot execute DistCp process: " + e, e); } finally { conf.setBoolean("mapred.mapper.new-api", false); } }
public DistCp run() { return new DistCp(userConf); } });
@Override public Job exec(Configuration conf, DistCpOptions options) throws Exception { return new DistCp(conf, options).execute(); } };
@Override public Job exec(Configuration conf, DistCpOptions options) throws Exception { return new DistCp(conf, options).execute(); } };
/** copy files from local file system to local file system */ public void testCopyFromLocalToLocal() throws Exception { Configuration conf = new Configuration(); FileSystem localfs = FileSystem.get(LOCAL_FS, conf); MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat"); ToolRunner.run(new DistCp(new Configuration()), new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat", "file:///"+TEST_ROOT_DIR+"/destdat"}); assertTrue("Source and destination directories do not match.", checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files)); deldir(localfs, TEST_ROOT_DIR+"/destdat"); deldir(localfs, TEST_ROOT_DIR+"/srcdat"); }
private static void assertRunDistCp(int exitCode, String src, String dst, String[] options, Configuration conf) throws Exception { DistCp distCp = new DistCp(conf, null); String[] optsArr = new String[options.length + 2]; System.arraycopy(options, 0, optsArr, 0, options.length); optsArr[optsArr.length - 2] = src; optsArr[optsArr.length - 1] = dst; assertEquals(exitCode, ToolRunner.run(conf, distCp, optsArr)); } }
private static void assertRunDistCp(int exitCode, String src, String dst, String[] options, Configuration conf) throws Exception { DistCp distCp = new DistCp(conf, null); String[] optsArr = new String[options.length + 2]; System.arraycopy(options, 0, optsArr, 0, options.length); optsArr[optsArr.length - 2] = src; optsArr[optsArr.length - 1] = dst; assertEquals(exitCode, ToolRunner.run(conf, distCp, optsArr)); } }
private static void invokeCopy(Configuration config, String[] parsedArgs) { try { log.info("Running DistCp with arguments [" + StringUtils.arrayToCommaDelimitedString(parsedArgs) + "]"); DistCpOptions inputOptions = OptionsParser.parse(parsedArgs); org.apache.hadoop.tools.DistCp distCp = new org.apache.hadoop.tools.DistCp(config, inputOptions); distCp.execute(); } catch (Exception e) { throw new HadoopException("Error running DistCp job", e); } }
/** * Runs distcp from /src to specified destination, preserving ACLs. Asserts * expected exit code. * * @param int exitCode expected exit code * @param dst String distcp destination * @throws Exception if there is any error */ private static void assertRunDistCp(int exitCode, String dst) throws Exception { DistCp distCp = new DistCp(conf, null); assertEquals(exitCode, ToolRunner.run( conf, distCp, new String[] { "-pa", "/src", dst })); }
@Test public void testSourceRoot() throws Exception { FileSystem fs = cluster.getFileSystem(); String rootStr = fs.makeQualified(new Path("/")).toString(); String testRoot = "/testdir." + getMethodName(); // Case 1. The target does not exist. Path tgtPath = new Path(testRoot + "/nodir"); String tgtStr = fs.makeQualified(tgtPath).toString(); String[] args = new String[]{rootStr, tgtStr}; Assert.assertThat(ToolRunner.run(conf, new DistCp(), args), is(0)); // Case 2. The target exists. Path tgtPath2 = new Path(testRoot + "/dir"); assertTrue(fs.mkdirs(tgtPath2)); String tgtStr2 = fs.makeQualified(tgtPath2).toString(); String[] args2 = new String[]{rootStr, tgtStr2}; Assert.assertThat(ToolRunner.run(conf, new DistCp(), args2), is(0)); } }
private void copyAndVerify(final DistributedFileSystem fs, final FileEntry[] srcFiles, final FileStatus[] srcStats, final String testDst, final String[] args) throws Exception { final String testRoot = "/testdir"; FsShell shell = new FsShell(fs.getConf()); LOG.info("ls before distcp"); LOG.info(execCmd(shell, "-lsr", testRoot)); LOG.info("_____ running distcp: " + args[0] + " " + args[1]); ToolRunner.run(conf, new DistCp(), args); LOG.info("ls after distcp"); LOG.info(execCmd(shell, "-lsr", testRoot)); FileStatus[] dstStat = getFileStatus(fs, testDst, srcFiles); for (int i=0; i< dstStat.length; i++) { compareFiles(fs, srcStats[i], dstStat[i]); } }
private void runTest(Path listFile, Path target, boolean targetExists, boolean sync) throws IOException { DistCpOptions options = new DistCpOptions(listFile, target); options.setSyncFolder(sync); options.setTargetPathExists(targetExists); try { new DistCp(getConf(), options).execute(); } catch (Exception e) { LOG.error("Exception encountered ", e); throw new IOException(e); } }
/** * test methods run end execute of DistCp class. silple copy file * @throws Exception */ @Test public void testCleanup() throws Exception { Configuration conf = getConf(); Path stagingDir = JobSubmissionFiles.getStagingDir(new Cluster(conf), conf); stagingDir.getFileSystem(conf).mkdirs(stagingDir); Path soure = createFile("tmp.txt"); Path target = createFile("target.txt"); DistCp distcp = new DistCp(conf, null); String[] arg = { soure.toString(), target.toString() }; distcp.run(arg); Assert.assertTrue(fs.exists(target)); }
/** * test methods run end execute of DistCp class. silple copy file * @throws Exception */ @Test public void testCleanup() throws Exception { Configuration conf = getConf(); Path stagingDir = JobSubmissionFiles.getStagingDir(new Cluster(conf), conf); stagingDir.getFileSystem(conf).mkdirs(stagingDir); Path soure = createFile("tmp.txt"); Path target = createFile("target.txt"); DistCp distcp = new DistCp(conf, null); String[] arg = { soure.toString(), target.toString() }; distcp.run(arg); Assert.assertTrue(fs.exists(target)); }
private void runTest(Path listFile, Path target, boolean targetExists, boolean sync) throws IOException { final DistCpOptions options = new DistCpOptions.Builder(listFile, target) .withSyncFolder(sync) .build(); try { final DistCp distcp = new DistCp(getConf(), options); distcp.context.setTargetPathExists(targetExists); distcp.execute(); } catch (Exception e) { LOG.error("Exception encountered ", e); throw new IOException(e); } }
private void runTest(Path listFile, Path target, boolean targetExists, boolean sync, boolean delete, boolean overwrite) throws IOException { DistCpOptions options = new DistCpOptions(listFile, target); options.setSyncFolder(sync); options.setDeleteMissing(delete); options.setOverwrite(overwrite); options.setTargetPathExists(targetExists); options.setNumListstatusThreads(numListstatusThreads); try { new DistCp(getConf(), options).execute(); } catch (Exception e) { LOG.error("Exception encountered ", e); throw new IOException(e); } }
/** * Executes DistCp and asserts that the job finished successfully. * * @param src source path * @param dst destination path * @throws Exception if there is a failure */ private void runDistCp(Path src, Path dst) throws Exception { DistCpOptions options = new DistCpOptions(Arrays.asList(src), dst); Job job = new DistCp(conf, options).execute(); assertNotNull("Unexpected null job returned from DistCp execution.", job); assertTrue("DistCp job did not complete.", job.isComplete()); assertTrue("DistCp job did not complete successfully.", job.isSuccessful()); }
private void runTest(Path listFile, Path target, boolean targetExists, boolean sync, boolean delete, boolean overwrite) throws IOException { final DistCpOptions options = new DistCpOptions.Builder(listFile, target) .withSyncFolder(sync) .withDeleteMissing(delete) .withOverwrite(overwrite) .withNumListstatusThreads(numListstatusThreads) .build(); try { final DistCp distCp = new DistCp(getConf(), options); distCp.context.setTargetPathExists(targetExists); distCp.execute(); } catch (Exception e) { LOG.error("Exception encountered ", e); throw new IOException(e); } }
/** * Run the distcp job. * @param optons distcp options * @return the job. It will have already completed. * @throws Exception failure */ private Job runDistCp(final DistCpOptions options) throws Exception { Job job = new DistCp(conf, options).execute(); assertNotNull("Unexpected null job returned from DistCp execution.", job); assertTrue("DistCp job did not complete.", job.isComplete()); assertTrue("DistCp job did not complete successfully.", job.isSuccessful()); return job; }