/** * Public Constructor. Creates DistCp object with specified input-parameters. * (E.g. source-paths, target-location, etc.) * @param inputOptions Options (indicating source-paths, target-location.) * @param configuration The Hadoop configuration against which the Copy-mapper must run. * @throws Exception */ public DistCp(Configuration configuration, DistCpOptions inputOptions) throws Exception { Configuration config = new Configuration(configuration); config.addResource(DISTCP_DEFAULT_XML); setConf(config); this.inputOptions = inputOptions; this.metaFolder = createMetaFolderPath(); }
/** * Public Constructor. Creates DistCp object with specified input-parameters. * (E.g. source-paths, target-location, etc.) * @param configuration configuration against which the Copy-mapper must run * @param inputOptions Immutable options * @throws Exception */ public DistCp(Configuration configuration, DistCpOptions inputOptions) throws Exception { Configuration config = new Configuration(configuration); config.addResource(DISTCP_DEFAULT_XML); config.addResource(DISTCP_SITE_XML); setConf(config); if (inputOptions != null) { this.context = new DistCpContext(inputOptions); } this.metaFolder = createMetaFolderPath(); }
/** * Create and submit the mapreduce job. * @return The mapreduce job object that has been submitted */ public Job createAndSubmitJob() throws Exception { assert context != null; assert getConf() != null; Job job = null; try { synchronized(this) { //Don't cleanup while we are setting up. metaFolder = createMetaFolderPath(); jobFS = metaFolder.getFileSystem(getConf()); job = createJob(); } prepareFileListing(job); job.submit(); submitted = true; } finally { if (!submitted) { cleanup(); } } String jobID = job.getJobID().toString(); job.getConfiguration().set(DistCpConstants.CONF_LABEL_DISTCP_JOB_ID, jobID); LOG.info("DistCp job-id: " + jobID); return job; }
synchronized(this) { metaFolder = createMetaFolderPath(); jobFS = metaFolder.getFileSystem(getConf()); job = createJob();