} else { Path parentDir = new Path(context.getConfiguration().get("mapred.work.output.dir")); Path childPath = new Path(parentDir,FileOutputFormat.getUniqueName(new JobConf(context.getConfiguration()), context.getConfiguration().get("mapreduce.output.basename", "part")));
/** * Helper function to generate a {@link Path} for a file that is unique for * the task within the job output directory. * * <p>The path can be used to create custom files from within the map and * reduce tasks. The path name will be unique for each task. The path parent * will be the job output directory.</p>ls * * <p>This method uses the {@link #getUniqueName} method to make the file name * unique for the task.</p> * * @param conf the configuration for the job. * @param name the name for the file. * @return a unique path accross all tasks of the job. */ public static Path getPathForCustomFile(JobConf conf, String name) { return new Path(getWorkOutputPath(conf), getUniqueName(conf, name)); } }
/** * Helper function to generate a {@link Path} for a file that is unique for * the task within the job output directory. * * <p>The path can be used to create custom files from within the map and * reduce tasks. The path name will be unique for each task. The path parent * will be the job output directory.</p>ls * * <p>This method uses the {@link #getUniqueName} method to make the file name * unique for the task.</p> * * @param conf the configuration for the job. * @param name the name for the file. * @return a unique path accross all tasks of the job. */ public static Path getPathForCustomFile(JobConf conf, String name) { return new Path(getWorkOutputPath(conf), getUniqueName(conf, name)); } }
/** * Helper function to generate a {@link Path} for a file that is unique for * the task within the job output directory. * * <p>The path can be used to create custom files from within the map and * reduce tasks. The path name will be unique for each task. The path parent * will be the job output directory.</p>ls * * <p>This method uses the {@link #getUniqueName} method to make the file name * unique for the task.</p> * * @param conf the configuration for the job. * @param name the name for the file. * @return a unique path accross all tasks of the job. */ public static Path getPathForCustomFile(JobConf conf, String name) { return new Path(getWorkOutputPath(conf), getUniqueName(conf, name)); } }
/** * Helper function to generate a {@link Path} for a file that is unique for * the task within the job output directory. * * <p>The path can be used to create custom files from within the map and * reduce tasks. The path name will be unique for each task. The path parent * will be the job output directory.</p>ls * * <p>This method uses the {@link #getUniqueName} method to make the file name * unique for the task.</p> * * @param conf the configuration for the job. * @param name the name for the file. * @return a unique path accross all tasks of the job. */ public static Path getPathForCustomFile(JobConf conf, String name) { return new Path(getWorkOutputPath(conf), getUniqueName(conf, name)); } }
/** * Helper function to generate a {@link Path} for a file that is unique for * the task within the job output directory. * * <p>The path can be used to create custom files from within the map and * reduce tasks. The path name will be unique for each task. The path parent * will be the job output directory.</p>ls * * <p>This method uses the {@link #getUniqueName} method to make the file name * unique for the task.</p> * * @param conf the configuration for the job. * @param name the name for the file. * @return a unique path accross all tasks of the job. */ public static Path getPathForCustomFile(JobConf conf, String name) { return new Path(getWorkOutputPath(conf), getUniqueName(conf, name)); } }
/** * Helper function to generate a {@link Path} for a file that is unique for * the task within the job output directory. * * <p>The path can be used to create custom files from within the map and * reduce tasks. The path name will be unique for each task. The path parent * will be the job output directory.</p>ls * * <p>This method uses the {@link #getUniqueName} method to make the file name * unique for the task.</p> * * @param conf the configuration for the job. * @param name the name for the file. * @return a unique path accross all tasks of the job. */ public static Path getPathForCustomFile(JobConf conf, String name) { return new Path(getWorkOutputPath(conf), getUniqueName(conf, name)); } }
/** * Helper function to generate a {@link Path} for a file that is unique for * the task within the job output directory. * * <p>The path can be used to create custom files from within the map and * reduce tasks. The path name will be unique for each task. The path parent * will be the job output directory.</p>ls * * <p>This method uses the {@link #getUniqueName} method to make the file name * unique for the task.</p> * * @param conf the configuration for the job. * @param name the name for the file. * @return a unique path accross all tasks of the job. */ public static Path getPathForCustomFile(JobConf conf, String name) { return new Path(getWorkOutputPath(conf), getUniqueName(conf, name)); } }
/** * Get the base Hadoop RecordWriter. * @param taskAttemptContext TaskAttemptContext * @param baseOutputFormat Hadoop OutputFormat * @return RecordWriter * @throws IOException Hadoop issues */ // CHECKSTYLE: stop LineLengthCheck protected static org.apache.hadoop.mapred.RecordWriter<WritableComparable, Writable> getBaseRecordWriter( TaskAttemptContext taskAttemptContext, org.apache.hadoop.mapred.OutputFormat baseOutputFormat) throws IOException { // CHECKSTYLE: resume LineLengthCheck JobConf jobConf = new JobConf(taskAttemptContext.getConfiguration()); int fileId = CREATED_FILES_COUNTER.incrementAndGet(); String name = FileOutputFormat.getUniqueName(jobConf, "part-" + fileId); if (LOG.isInfoEnabled()) { LOG.info("getBaseRecordWriter: Created new with file " + name); } Reporter reporter = new ProgressReporter(taskAttemptContext); return baseOutputFormat.getRecordWriter(null, jobConf, name, reporter); }
/** * Get the base Hadoop RecordWriter. * @param taskAttemptContext TaskAttemptContext * @param baseOutputFormat Hadoop OutputFormat * @return RecordWriter * @throws IOException Hadoop issues */ // CHECKSTYLE: stop LineLengthCheck protected static org.apache.hadoop.mapred.RecordWriter<WritableComparable, Writable> getBaseRecordWriter( TaskAttemptContext taskAttemptContext, org.apache.hadoop.mapred.OutputFormat baseOutputFormat) throws IOException { // CHECKSTYLE: resume LineLengthCheck HadoopUtils.setWorkOutputDir(taskAttemptContext); JobConf jobConf = new JobConf(taskAttemptContext.getConfiguration()); int fileId = CREATED_FILES_COUNTER.incrementAndGet(); String name = FileOutputFormat.getUniqueName(jobConf, "part-" + fileId); Reporter reporter = new ProgressReporter(taskAttemptContext); org.apache.hadoop.mapred.RecordWriter<WritableComparable, Writable> baseWriter = baseOutputFormat.getRecordWriter(null, jobConf, name, reporter); LOG.info("getBaseRecordWriter: Created new {} with file {}", baseWriter, name); return baseWriter; }
} else { Path parentDir = new Path(context.getConfiguration().get("mapred.work.output.dir")); Path childPath = new Path(parentDir,FileOutputFormat.getUniqueName(new JobConf(context.getConfiguration()), context.getConfiguration().get("mapreduce.output.basename", "part")));
} else { Path parentDir = new Path(context.getConfiguration().get("mapred.work.output.dir")); Path childPath = new Path(parentDir,FileOutputFormat.getUniqueName(new JobConf(context.getConfiguration()), "part"));
} else { Path parentDir = new Path(context.getConfiguration().get("mapred.work.output.dir")); Path childPath = new Path(parentDir,FileOutputFormat.getUniqueName(new JobConf(context.getConfiguration()), context.getConfiguration().get("mapreduce.output.basename", "part")));
} else { Path parentDir = new Path(context.getConfiguration().get("mapred.work.output.dir")); Path childPath = new Path(parentDir,FileOutputFormat.getUniqueName(new JobConf(context.getConfiguration()), "part"));