Refine search
groupByJob = Job.getInstance( new Configuration(), StringUtils.format("%s-determine_partitions_groupby-%s", config.getDataSource(), config.getIntervals()) ); config.addJobProperties(groupByJob); groupByJob.setMapperClass(DeterminePartitionsGroupByMapper.class); groupByJob.setMapOutputKeyClass(BytesWritable.class); groupByJob.setMapOutputValueClass(NullWritable.class); groupByJob.setCombinerClass(DeterminePartitionsGroupByReducer.class); if (groupByJob.getJobID() != null) { JobHelper.writeJobIdToFile(config.getHadoopJobIdFileName(), groupByJob.getJobID().toString()); dimSelectionJob.getConfiguration().set("io.sort.record.percent", "0.19"); if (dimSelectionJob.getJobID() != null) { JobHelper.writeJobIdToFile(config.getHadoopJobIdFileName(), dimSelectionJob.getJobID().toString()); if (!dimSelectionJob.waitForCompletion(true)) { log.error("Job failed: %s", dimSelectionJob.getJobID().toString()); failureCause = Utils.getFailureMessage(dimSelectionJob, config.JSON_MAPPER); return false; final Path partitionInfoPath = config.makeSegmentPartitionInfoPath(segmentGranularity); if (fileSystem == null) { fileSystem = partitionInfoPath.getFileSystem(dimSelectionJob.getConfiguration());
this.configuration.set("mapreduce.output.basename", "tmp"); TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_" + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s", " ").replace(" ", "0") + Integer.toString(taskNumber + 1) + "_0"); this.configuration.set("mapred.task.id", taskAttemptID.toString()); this.configuration.setInt("mapred.task.partition", taskNumber + 1); this.configuration.set("mapreduce.task.attempt.id", taskAttemptID.toString()); this.configuration.setInt("mapreduce.task.partition", taskNumber + 1); this.context = new TaskAttemptContextImpl(this.configuration, taskAttemptID); this.outputCommitter = this.mapreduceOutputFormat.getOutputCommitter(this.context); this.outputCommitter.setupJob(new JobContextImpl(this.configuration, new JobID())); } catch (Exception e) { throw new RuntimeException(e); this.configuration.set("mapreduce.task.output.dir", ((FileOutputCommitter) this.outputCommitter).getWorkPath().toString());
/** {@inheritDoc} */ @Override public JobStatus getJobStatus(JobID jobId) throws IOException, InterruptedException { try { Long delay = conf.getLong(HadoopJobProperty.JOB_STATUS_POLL_DELAY.propertyName(), -1); HadoopJobStatus status; if (delay >= 0) status = execute(HadoopProtocolJobStatusTask.class, jobId.getJtIdentifier(), jobId.getId(), delay); else status = execute(HadoopProtocolJobStatusTask.class, jobId.getJtIdentifier(), jobId.getId()); if (status == null) throw new IOException("Job tracker doesn't have any information about the job: " + jobId); return processStatus(status); } catch (GridClientException e) { throw new IOException("Failed to get job status: " + jobId, e); } }
@Override public TaskAttemptID newTaskAttemptID(JobID jobId, boolean isMap, int taskId, int id) { return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(), isMap ? TaskType.MAP : TaskType.REDUCE, taskId, id); }
/** {@inheritDoc} */ @Override public void killJob(JobID jobId) throws IOException, InterruptedException { try { execute(HadoopProtocolKillJobTask.class, jobId.getJtIdentifier(), jobId.getId()); } catch (GridClientException e) { throw new IOException("Failed to kill job: " + jobId, e); } }
job = Job.getInstance( new Configuration(), StringUtils.format("%s-index-generator-%s", config.getDataSource(), config.getIntervals()) ); job.getConfiguration().set("io.sort.record.percent", "0.23"); JobHelper.injectDruidProperties(job.getConfiguration(), config.getAllowedHadoopPrefix()); job.setMapperClass(IndexGeneratorMapper.class); if (job.getJobID() != null) { JobHelper.writeJobIdToFile(config.getHadoopJobIdFileName(), job.getJobID().toString());
conf.set(JAR_ARGS_NAME, TempletonUtils.encodeArray(args)); String memoryMb = appConf.mapperMemoryMb(); if(memoryMb != null && memoryMb.length() != 0) { conf.set(AppConfig.HADOOP_MAP_MEMORY_MB, memoryMb); conf.set(AppConfig.HADOOP_MR_AM_MEMORY_MB, amMemoryMB); conf.set("user.name", user); job = new Job(conf); job.setJarByClass(LaunchMapper.class); job.setJobName(TempletonControllerJob.class.getSimpleName()); job.setMapperClass(LaunchMapper.class); job.setMapOutputKeyClass(Text.class); job.setNumReduceTasks(0); JobClient jc = new JobClient(new JobConf(job.getConfiguration())); submittedJobId.toString(), metastoreTokenStrForm); LOG.debug("Added metastore delegation token for jobId=" + submittedJobId.toString() + " user=" + user);
@Test public void testMaxBlockLocationsOldSplits() throws Exception { TEST_DIR.mkdirs(); try { Configuration conf = new Configuration(); conf.setInt(MRConfig.MAX_BLOCK_LOCATIONS_KEY, 4); Path submitDir = new Path(TEST_DIR.getAbsolutePath()); FileSystem fs = FileSystem.getLocal(conf); org.apache.hadoop.mapred.FileSplit split = new org.apache.hadoop.mapred.FileSplit(new Path("/some/path"), 0, 1, new String[] { "loc1", "loc2", "loc3", "loc4", "loc5" }); JobSplitWriter.createSplitFiles(submitDir, conf, fs, new org.apache.hadoop.mapred.InputSplit[] { split }); JobSplit.TaskSplitMetaInfo[] infos = SplitMetaInfoReader.readSplitMetaInfo(new JobID(), fs, conf, submitDir); assertEquals("unexpected number of splits", 1, infos.length); assertEquals("unexpected number of split locations", 4, infos[0].getLocations().length); } finally { FileUtil.fullyDelete(TEST_DIR); } } }
static Path getTxFile(Configuration configuration, @Nullable JobID jobID) throws IOException { Path stagingDir = MRApps.getStagingAreaDir(configuration, UserGroupInformation.getCurrentUser().getShortUserName()); int appAttemptId = configuration.getInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 0); // following the pattern of other files in the staging dir String jobId = jobID != null ? jobID.toString() : configuration.get(MRJobConfig.ID); return new Path(stagingDir, jobId + "/" + jobId + "_" + appAttemptId + "_tx-file"); }
CompactionPathParser.CompactionParserResult result = new CompactionPathParser(state).parse(dataset); Path tmpPath = configurator.getMrOutputPath(); Path dstPath = new Path (result.getDstAbsoluteDir()); long oldTotalRecords = helper.readRecordCount(new Path (result.getDstAbsoluteDir())); long executeCount = helper.readExecutionCount (new Path (result.getDstAbsoluteDir())); Counter counter = job.getCounters().findCounter(AvroKeyMapper.EVENT_COUNTER.RECORD_COUNT); newTotalRecords = counter.getValue(); compactState.setProp(CompactionSlaEventHelper.RECORD_COUNT_TOTAL, Long.toString(newTotalRecords)); compactState.setProp(CompactionSlaEventHelper.EXEC_COUNT_TOTAL, Long.toString(executeCount + 1)); compactState.setProp(CompactionSlaEventHelper.MR_JOB_ID, this.configurator.getConfiguredJob().getJobID().toString()); helper.saveState(new Path (result.getDstAbsoluteDir()), compactState); CompactionSlaEventHelper.PREV_RECORD_COUNT_TOTAL, Long.toString(oldTotalRecords), CompactionSlaEventHelper.EXEC_COUNT_TOTAL, Long.toString(executeCount + 1), CompactionSlaEventHelper.MR_JOB_ID, this.configurator.getConfiguredJob().getJobID().toString()); this.eventSubmitter.submit(CompactionSlaEventHelper.COMPACTION_RECORD_COUNT_EVENT, eventMetadataMap);
throws IOException, InterruptedException, ClassNotFoundException { Job job = Job.getInstance(hadoopConf); job.setJarByClass(mapperClass); job.setJobName(jobName); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(NullWritable.class); job.getJobID().toString(), HadoopCompatLoader.DEFAULT_COMPAT.getJobFailureString(job)); } catch (RuntimeException e) { f = "Job failed (unable to read job status programmatically -- see MapReduce logs for information)";
private void cleanupTempFiles(JobContext context) { try { Configuration conf = context.getConfiguration(); Path targetWorkPath = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH)); FileSystem targetFS = targetWorkPath.getFileSystem(conf); String jobId = context.getJobID().toString(); deleteAttemptTempFiles(targetWorkPath, targetFS, jobId); deleteAttemptTempFiles(targetWorkPath.getParent(), targetFS, jobId); } catch (Throwable t) { LOG.warn("Unable to cleanup temp files", t); } }
for (Path aSrc : srcs) { totalSrcLgth += BackupUtils.getFilesLength(aSrc.getFileSystem(super.getConf()), aSrc); MapReduceBackupCopyJob.this.getConf().getInt("hbase.backup.progressreport.frequency", 500); float lastProgress = progressDone; while (!job.isComplete()) { float newProgress = progressDone + job.mapProgress() * subTaskPercntgInWholeTask * (1 - INIT_PROGRESS); progressDone + job.mapProgress() * subTaskPercntgInWholeTask * (1 - INIT_PROGRESS); BigDecimal progressData = new BigDecimal(newProgress * 100).setScale(1, BigDecimal.ROUND_HALF_UP); String jobID = job.getJobID().toString(); job.getConfiguration().set(DistCpConstants.CONF_LABEL_DISTCP_JOB_ID, jobID); LOG.debug("DistCp job-id: " + jobID + " completed: " + job.isComplete() + " "
@Test (timeout=5000) public void testJobInfo() throws IOException { JobID jid = new JobID("001", 1); Text user = new Text("User"); Path path = new Path("/tmp/test"); JobInfo info = new JobInfo(jid, user, path); ByteArrayOutputStream out = new ByteArrayOutputStream(); info.write(new DataOutputStream(out)); JobInfo copyinfo = new JobInfo(); copyinfo.readFields(new DataInputStream(new ByteArrayInputStream(out .toByteArray()))); assertEquals(info.getJobID().toString(), copyinfo.getJobID().toString()); assertEquals(info.getJobSubmitDir().getName(), copyinfo.getJobSubmitDir() .getName()); assertEquals(info.getUser().toString(), copyinfo.getUser().toString()); }
@Test public void testGetJobWithRetry() throws Exception { Configuration conf = new Configuration(); conf.setInt(MRJobConfig.MR_CLIENT_JOB_MAX_RETRIES, 1); final Cluster mockCluster = mock(Cluster.class); final Job mockJob = Job.getInstance(conf); when(mockCluster.getJob(any(JobID.class))).thenReturn( null).thenReturn(mockJob); CLI cli = new CLI(conf); cli.cluster = mockCluster; Job job = cli.getJob(JobID.forName("job_1234654654_001")); Assert.assertTrue("job is null", job != null); }
PrintWriter writer = null; try { String enc = conf.get(AppConfig.EXEC_ENCODING_NAME); InputStreamReader isr = new InputStreamReader(in, enc); BufferedReader reader = new BufferedReader(isr); String percent = TempletonUtils.extractPercentComplete(line); String childid = TempletonUtils.extractChildJobId(line); updateJobStatePercentAndChildId(conf, jobid.toString(), percent, childid);
Configuration conf = new Configuration(HadoopUtil.getCurrentConfiguration()); overwriteJobConf(conf, executableContext.getConfig(), getMapReduceParams().trim().split("\\s+")); Job job = new Cluster(conf).getJob(JobID.forName(mrJobId)); if (job == null || job.getJobState() == JobStatus.State.FAILED) {
final Map<String, String> extra = executableManager.getOutput(getId()).getExtra(); if (extra.containsKey(ExecutableConstants.MR_JOB_ID)) { job = new Cluster(new Configuration()).getJob(JobID.forName(extra.get(ExecutableConstants.MR_JOB_ID))); logger.info("mr_job_id:" + extra.get(ExecutableConstants.MR_JOB_ID + " resumed")); } else {