public void cleanupJob(JobContext context) throws IOException { JobConf conf = context.getJobConf(); // do the clean up of temporary directory Path outputPath = FileOutputFormat.getOutputPath(conf); if (outputPath != null) { Path tmpDir = new Path(outputPath, FileOutputCommitter.TEMP_DIR_NAME); FileSystem fileSys = tmpDir.getFileSystem(conf); context.getProgressible().progress(); if (fileSys.exists(tmpDir)) { fileSys.delete(tmpDir, true); } } }
@Override @Deprecated public void cleanupJob(JobContext context) throws IOException { JobConf conf = context.getJobConf(); // do the clean up of temporary directory Path outputPath = FileOutputFormat.getOutputPath(conf); if (outputPath != null) { Path tmpDir = new Path(outputPath, FileOutputCommitter.TEMP_DIR_NAME); FileSystem fileSys = tmpDir.getFileSystem(conf); context.getProgressible().progress(); if (fileSys.exists(tmpDir)) { fileSys.delete(tmpDir, true); } else { LOG.warn("Output Path is Null in cleanup"); } } }
@Override @Deprecated public void cleanupJob(JobContext context) throws IOException { JobConf conf = context.getJobConf(); // do the clean up of temporary directory Path outputPath = FileOutputFormat.getOutputPath(conf); if (outputPath != null) { Path tmpDir = new Path(outputPath, FileOutputCommitter.TEMP_DIR_NAME); FileSystem fileSys = tmpDir.getFileSystem(conf); context.getProgressible().progress(); if (fileSys.exists(tmpDir)) { if (!fileSys.delete(tmpDir, true)) { LOG.warn("Deleting output in " + outputPath + " returns false"); } } } else { LOG.warn("Output path is null in cleanup"); } }
private static Counter createCounter(JobContext context) { assert context != null; if (context instanceof Progressable) { return new ProgressableCounter((Progressable) context); } else if (context instanceof org.apache.hadoop.mapred.JobContext) { return new ProgressableCounter(((org.apache.hadoop.mapred.JobContext) context).getProgressible()); } else { return new Counter(); } }
private org.apache.hadoop.mapreduce.TaskAttemptContext toTaskAttemptContext(JobContext jobContext) { assert jobContext != null; // NOTE: JobContext can be instance of TaskAttemptContext. if (TaskAttemptContext.class.isInstance(jobContext)) { return TaskAttemptContext.class.cast(jobContext); } Progressable progressable = jobContext.getProgressible(); if (progressable == null) { LOG.warn(MessageFormat.format( "JobContext has no progressable object: {0}", jobContext.getClass().getName())); } if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format( "Progressable object is found (jobId={0}, object={1})", //$NON-NLS-1$ jobContext.getJobID(), progressable)); } TaskAttemptID id = new TaskAttemptID(new TaskID(jobContext.getJobID(), TaskType.MAP, 0), 0); return new TaskAttemptContextImpl(jobContext.getConfiguration(), id) { @Override public void progress() { progressable.progress(); super.progress(); } }; }