FileSystem fs = emptyScratchDir.getFileSystem(job); fs.mkdirs(emptyScratchDir); } catch (IOException e) { e.printStackTrace(); job.setOutputFormat(HiveOutputFormatImpl.class); job.setMapRunnerClass(ExecMapRunner.class); job.setMapperClass(ExecMapper.class); Path hdfsPath = mWork.getTmpHDFSPath(); hdfs.copyFromLocalFile(archivePath, hdfsFilePath); jc = new JobClient(job); rj = jc.submitJob(job); this.jobID = rj.getJobID(); updateStatusInQueryDisplay(); returnVal = jobExecHelper.progress(rj, jc, ctx); killJob(); jobID = rj.getID().toString(); jc.close();
reader = new BufferedReader(new InputStreamReader(xlearningProcess.getInputStream())); List<OutputInfo> outputs = Arrays.asList(amClient.getOutputLocation()); JobConf jobConf = new JobConf(conf); jobConf.setOutputKeyClass(Text.class); jobConf.setOutputValueClass(Text.class); jobConf.setBoolean("mapred.output.compress", true); jobConf.set("mapred.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec"); jobConf.setOutputFormat(TextMultiOutputFormat.class); Path remotePath = new Path(outputs.get(0).getDfsLocation() + "/_temporary/" + containerId.toString()); FileSystem dfs = remotePath.getFileSystem(jobConf); jobConf.set(XLearningConstants.STREAM_OUTPUT_DIR, remotePath.makeQualified(dfs).toString()); OutputFormat outputFormat = ReflectionUtils.newInstance(conf.getClass(XLearningConfiguration.XLEARNING_OUTPUTFORMAT_CLASS, XLearningConfiguration.DEFAULT_XLEARNING_OUTPUTF0RMAT_CLASS, OutputFormat.class), jobConf); outputFormat.checkOutputSpecs(dfs, jobConf); JobID jobID = new JobID(new SimpleDateFormat("yyyyMMddHHmm").format(new Date()), 0); TaskAttemptID taId = new TaskAttemptID(new TaskID(jobID, true, 0), 0); jobConf.set("mapred.tip.id", taId.getTaskID().toString()); jobConf.set("mapred.task.id", taId.toString()); jobConf.set("mapred.job.id", jobID.toString()); amClient.reportMapedTaskID(containerId, taId.toString()); RecordWriter writer = outputFormat.getRecordWriter(dfs, jobConf, "part-r", Reporter.NULL); dfs.close(); } catch (Exception e) { LOG.warn("Exception in thread stdoutRedirectThread");
private void logJob(String logDir, String jobID, PrintWriter listWriter) throws IOException { RunningJob rj = jobClient.getJob(JobID.forName(jobID)); String jobURLString = rj.getTrackingURL(); Path jobDir = new Path(logDir, jobID); fs.mkdirs(jobDir); logJobConf(jobID, jobURLString, jobDir.toString()); } catch (IOException e) { System.err.println("Cannot retrieve job.xml.html for " + jobID); listWriter.println("job: " + jobID + "(" + "name=" + rj.getJobName() + "," + "status=" + JobStatus.getJobRunState(rj.getJobState()) + ")"); logAttempt(jobID, attempt, jobDir.toString()); listWriter.println(" attempt:" + attempt.id + "(" + "type=" + attempt.type + "," + "status=" + attempt.status + ","
System.exit(1); JobConf conf = new JobConf(new Path(jobFilename.toString())); TaskAttemptID taskId = TaskAttemptID.forName(conf.get("mapred.task.id")); boolean isMap = conf.getBoolean("mapred.task.is.map", true); int partition = conf.getInt("mapred.task.partition", 0); FileSystem local = FileSystem.getLocal(conf); LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir"); File workDirName = new File(lDirAlloc.getLocalPathToRead( TaskTracker.getLocalTaskDir( taskId.getJobID().toString(), taskId.toString()) + Path.SEPARATOR + "work", conf). toString()); local.setWorkingDirectory(new Path(workDirName.toString())); FileSystem.get(conf).setWorkingDirectory(conf.getWorkingDirectory());
public Job(JobID jobid, String jobSubmitDir) throws IOException { this.systemJobDir = new Path(jobSubmitDir); this.systemJobFile = new Path(systemJobDir, "job.xml"); this.id = jobid; JobConf conf = new JobConf(systemJobFile); this.localFs = FileSystem.getLocal(conf); String user = UserGroupInformation.getCurrentUser().getShortUserName(); this.localJobDir = localFs.makeQualified(new Path( new Path(conf.getLocalPath(jobDir), user), jobid.toString())); this.localJobFile = new Path(this.localJobDir, id + ".xml"); OutputStream out = localFs.create(localJobFile); try { conf.writeXml(out); } finally { out.close();
private void localizeTask(Task task) throws IOException{ lDirAlloc.getLocalPathForWrite( TaskTracker.getLocalTaskDir(task.getJobID().toString(), task.getTaskID().toString(), task.isTaskCleanupTask()), defaultJobConf ); String workDir = lDirAlloc.getLocalPathToRead( TaskTracker.getLocalJobDir(task.getJobID().toString()) + Path.SEPARATOR + "work", defaultJobConf).toString(); String link = localTaskDir.getParent().toString() + Path.SEPARATOR + "work"; File flink = new File(link); Path cwd = lDirAlloc.getLocalPathForWrite( getLocalTaskDir(task.getJobID().toString(), task.getTaskID().toString(), task.isTaskCleanupTask()) + Path.SEPARATOR + MRConstants.WORKDIR, defaultJobConf); if (!localFs.mkdirs(cwd)) { localJobConf.set("mapred.local.dir", fConf.get("mapred.local.dir")); if (fConf.get("slave.host.name") != null) { localJobConf.set("mapred.task.id", task.getTaskID().toString());
protected ApplicationId getChildMRJobApplicationId(final Configuration conf) throws IOException { final List<ApplicationId> applicationIdList = new ArrayList<>(); final Path inputDir = new Path(getFsTestCaseDir(), "input"); final Path wfIDFile = new Path(inputDir, LauncherMainTester.JOB_ID_FILE_NAME); final FileSystem fs = FileSystem.get(conf); // wait until we have the running child MR job's ID from HDFS waitFor(JOB_TIMEOUT, new ApplicationIdExistsPredicate(fs, wfIDFile)); if (!fs.exists(wfIDFile) || !fs.isFile(wfIDFile)) { throw new IOException("Workflow ID file does not exist: " + wfIDFile.toString()); } try (final BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(wfIDFile)))) { final String line = reader.readLine(); JobID.forName(line); final String jobID = line; final String appID = jobID.replace("job", "application"); final ApplicationId id = ConverterUtils.toApplicationId(appID); applicationIdList.add(id); } assertTrue("Application ID should've been found. No external Child ID was found in " + wfIDFile.toString(), applicationIdList.size() == 1); return applicationIdList.get(0); }
private String runJob() throws Exception { OutputStream os = getFileSystem().create(new Path(getInputDir(), "text.txt")); Writer wr = new OutputStreamWriter(os); wr.write("hello1\n"); wr.write("hello2\n"); wr.write("hello3\n"); wr.close(); JobConf conf = createJobConf(); conf.setJobName("mr"); conf.setJobPriority(JobPriority.HIGH); conf.setInputFormat(TextInputFormat.class); conf.setMapOutputKeyClass(LongWritable.class); conf.setMapOutputValueClass(Text.class); conf.setOutputFormat(TextOutputFormat.class); conf.setOutputKeyClass(LongWritable.class); conf.setOutputValueClass(Text.class); conf.setMapperClass(org.apache.hadoop.mapred.lib.IdentityMapper.class); conf.setReducerClass(org.apache.hadoop.mapred.lib.IdentityReducer.class); FileInputFormat.setInputPaths(conf, getInputDir()); FileOutputFormat.setOutputPath(conf, getOutputDir()); return JobClient.runJob(conf).getID().toString(); }
throws IOException, InterruptedException { String jarFile = localJobConf.getJar(); FileStatus status = null; long jarFileSize = -1; if (jarFile != null) { Path jarFilePath = new Path(jarFile); FileSystem fs = getFS(jarFilePath, jobId, localJobConf); try { status = fs.getFileStatus(jarFilePath); jarFileSize = status.getLen(); } catch (FileNotFoundException fe) { lDirAlloc.getLocalPathForWrite( getJobJarFile(user, jobId.toString()), 5 * jarFileSize, fConf); fs.copyToLocalFile(jarFilePath, localJarFile); localJobConf.setJar(localJarFile.toString()); new File(localJarFile.toString()), new File(localJarFile.getParent().toString()), localJobConf.getJarUnpackPattern());
MyFakeJobInProgress(JobConf jc, JobTracker jt) throws IOException { super((jobid = new JobID(jtIdentifier, jobCounter ++)), jc, jt); Path jobFile = new Path("Dummy"); this.profile = new JobProfile(jc.getUser(), jobid, jobFile.toString(), null, jc.getJobName(), jc.getQueueName()); this.jobHistory = new FakeJobHistory(); }
private static void configureJobConf(Configuration conf, Map<String, LocalResource> localResources, final org.apache.hadoop.mapred.JobID oldJobId) throws IOException { // Set up JobConf to be localized properly on the remote NM. Path path = MRApps.getStagingAreaDir(conf, UserGroupInformation.getCurrentUser().getShortUserName()); Path remoteJobSubmitDir = new Path(path, oldJobId.toString()); Path remoteJobConfPath = new Path(remoteJobSubmitDir, MRJobConfig.JOB_CONF_FILE); FileSystem remoteFS = FileSystem.get(conf); // There is no point to ask task container's NM to upload the resource // to shared cache (job conf is not shared). Therefore, createLocalResource // will set the shared cache upload policy to false localResources.put(MRJobConfig.JOB_CONF_FILE, createLocalResource(remoteFS, remoteJobConfPath, null, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION)); LOG.info("The job-conf file on the remote FS is " + remoteJobConfPath.toUri().toASCIIString()); }
/** * Download the job-token file from the FS and save on local fs. * @param user * @param jobId * @param jobConf * @return the local file system path of the downloaded file. * @throws IOException */ private String localizeJobTokenFile(String user, JobID jobId) throws IOException { // check if the tokenJob file is there.. Path skPath = new Path(systemDirectory, jobId.toString()+"/"+TokenCache.JOB_TOKEN_HDFS_FILE); FileStatus status = null; long jobTokenSize = -1; status = systemFS.getFileStatus(skPath); //throws FileNotFoundException jobTokenSize = status.getLen(); Path localJobTokenFile = lDirAlloc.getLocalPathForWrite(getLocalJobTokenFile(user, jobId.toString()), jobTokenSize, fConf); String localJobTokenFileStr = localJobTokenFile.toUri().getPath(); LOG.debug("localizingJobTokenFile from sd="+skPath.toUri().getPath() + " to " + localJobTokenFileStr); // Download job_token systemFS.copyToLocalFile(skPath, localJobTokenFile); return localJobTokenFileStr; }
private Task createDummyTask(TaskType type) throws IOException, ClassNotFoundException, InterruptedException { JobConf conf = new JobConf(); conf.setOutputCommitter(CommitterThatAlwaysRequiresCommit.class); Path outDir = new Path(rootDir, "output"); FileOutputFormat.setOutputPath(conf, outDir); JobID jobId = JobID.forName("job_201002121132_0001"); Task testTask; if (type == TaskType.MAP) { testTask = new MapTask(); } else { testTask = new ReduceTask(); } testTask.setConf(conf); testTask.initialize(conf, jobId, Reporter.NULL, false); return testTask; }
void localizeTask(Task task) throws IOException{ FileSystem localFs = FileSystem.getLocal(fConf); task.getJobID().toString(), task.getTaskID().toString(), task.isTaskCleanupTask()); lDirAlloc.getLocalPathForWrite(getTaskWorkDir(task.getUser(), task .getJobID().toString(), task.getTaskID().toString(), task .isTaskCleanupTask()), defaultJobConf); if (!localFs.mkdirs(cwd)) { throw new IOException("Mkdirs failed to create " + cwd.toString()); localJobConf.set(LOCAL_DIR, fConf.get(LOCAL_DIR)); if (fConf.get(TT_HOST_NAME) != null) { localJobConf.set(TT_HOST_NAME, fConf.get(TT_HOST_NAME)); if (keepPattern != null) { alwaysKeepTaskFiles = Pattern.matches(keepPattern, task.getTaskID().toString()); } else { alwaysKeepTaskFiles = false;
int curDirNumber, int obsoleteDirNumber, HiveConf hiveConf, IMetaStoreClient msc, long id, String jobName) throws IOException { job.setBoolean(IS_MAJOR, compactionType == CompactionType.MAJOR); if(dirsToSearch == null) { dirsToSearch = new StringableList(); if (baseDir != null) job.set(BASE_DIR, baseDir.toString()); job.set(DELTA_DIRS, deltaDirs.toString()); job.set(DIRS_TO_SEARCH, dirsToSearch.toString()); job.setLong(MIN_TXN, minTxn); JobClient jc = null; try { jc = new JobClient(job); RunningJob rj = jc.submitJob(job); LOG.info("Submitted compaction job '" + job.getJobName() + "' with jobID=" + rj.getID() + " compaction ID=" + id); try { msc.setHadoopJobid(rj.getID().toString(), id); } catch (TException e) { LOG.warn("Error setting hadoop job, jobId=" + rj.getID().toString() + " compactionId=" + id, e); jc.close();
private static void executeJavaMapReduce(String[] args) throws IOException, InterruptedException { JobConf jConf = createSleepMapperReducerJobConf(); final Path input = new Path(args[1]); FileInputFormat.setInputPaths(jConf, input); FileOutputFormat.setOutputPath(jConf, new Path(args[2])); writeToFile(input, jConf, "dummy\n", "data.txt"); JobClient jc = new JobClient(jConf); System.out.println("Submitting MR job"); RunningJob job = jc.submitJob(jConf); System.out.println("Submitted job " + job.getID().toString()); writeToFile(input, jConf, job.getID().toString(), JOB_ID_FILE_NAME); job.waitForCompletion(); jc.monitorAndPrintJob(jConf, job); if (job.getJobState() != JobStatus.SUCCEEDED) { System.err.println(job.getJobState() + " job state instead of" + JobStatus.SUCCEEDED); System.exit(-1); } }
/** * Localize the given JobConf to be specific for this task. */ public void localizeConfiguration(JobConf conf) throws IOException { conf.set(JobContext.TASK_ID, taskId.getTaskID().toString()); conf.set(JobContext.TASK_ATTEMPT_ID, taskId.toString()); conf.setBoolean(JobContext.TASK_ISMAP, isMapTask()); conf.setInt(JobContext.TASK_PARTITION, partition); conf.set(JobContext.ID, taskId.getJobID().toString()); }
public void checkAndAddJob(FileStatus status) throws IOException { String jobName = status.getPath().getName(); if (isJobNameValid(jobName)) { if (JobClient.isJobDirValid(status.getPath(), fs)) { recoveryManager.addJobForRecovery(JobID.forName(jobName)); } else { LOG.info("Found an incomplete job directory " + jobName + "." + " Deleting it!!"); fs.delete(status.getPath(), true); } } else { LOG.info("Deleting " + status.getPath()); fs.delete(status.getPath(), true); } }
/** * Download the job configuration file from the FS. * * @param t Task whose job file has to be downloaded * @param jobId jobid of the task * @return the local file system path of the downloaded file. * @throws IOException */ private Path localizeJobConfFile(Path jobFile, String user, JobID jobId) throws IOException, InterruptedException { final JobConf conf = new JobConf(getJobConf()); FileSystem userFs = getFS(jobFile, jobId, conf); // Get sizes of JobFile // sizes are -1 if they are not present. FileStatus status = null; long jobFileSize = -1; try { status = userFs.getFileStatus(jobFile); jobFileSize = status.getLen(); } catch(FileNotFoundException fe) { jobFileSize = -1; } Path localJobFile = lDirAlloc.getLocalPathForWrite(getLocalJobConfFile(user, jobId.toString()), jobFileSize, fConf); // Download job.xml userFs.copyToLocalFile(jobFile, localJobFile); return localJobFile; }
/** Creates the working directory pathname for a task attempt. */ static File formWorkDir(LocalDirAllocator lDirAlloc, TaskAttemptID task, boolean isCleanup, JobConf conf) throws IOException { Path workDir = lDirAlloc.getLocalPathToRead(TaskTracker.getTaskWorkDir( conf.getUser(), task.getJobID().toString(), task.toString(), isCleanup), conf); return new File(workDir.toString()); }