/** * Helper function to generate a {@link Path} for a file that is unique for * the task within the job output directory. * * <p>The path can be used to create custom files from within the map and * reduce tasks. The path name will be unique for each task. The path parent * will be the job output directory.</p>ls * * <p>This method uses the {@link #getUniqueName} method to make the file name * unique for the task.</p> * * @param conf the configuration for the job. * @param name the name for the file. * @return a unique path accross all tasks of the job. */ public static Path getPathForCustomFile(JobConf conf, String name) { return new Path(getWorkOutputPath(conf), getUniqueName(conf, name)); } }
/** * Helper function to generate a {@link Path} for a file that is unique for * the task within the job output directory. * * <p>The path can be used to create custom files from within the map and * reduce tasks. The path name will be unique for each task. The path parent * will be the job output directory.</p>ls * * <p>This method uses the {@link #getUniqueName} method to make the file name * unique for the task.</p> * * @param conf the configuration for the job. * @param name the name for the file. * @return a unique path accross all tasks of the job. */ public static Path getPathForCustomFile(JobConf conf, String name) { return new Path(getWorkOutputPath(conf), getUniqueName(conf, name)); } }
public void configure(JobConf conf) { this.conf = conf; tmpOutputDir = FileOutputFormat.getWorkOutputPath(this.conf); masterIndex = new Path(tmpOutputDir, "_masterindex"); index = new Path(tmpOutputDir, "_index"); replication = conf.getInt(HAR_REPLICATION_LABEL, 3); try { fs = masterIndex.getFileSystem(conf); fs.delete(masterIndex, false); fs.delete(index, false); indexStream = fs.create(index); outStream = fs.create(masterIndex); String version = VERSION + " \n"; outStream.write(version.getBytes(Charsets.UTF_8)); } catch(IOException e) { throw new RuntimeException(e); } }
public void configure(JobConf conf) { this.conf = conf; tmpOutputDir = FileOutputFormat.getWorkOutputPath(this.conf); masterIndex = new Path(tmpOutputDir, "_masterindex"); index = new Path(tmpOutputDir, "_index"); try { fs = masterIndex.getFileSystem(conf); if (fs.exists(masterIndex)) { fs.delete(masterIndex, false); } if (fs.exists(index)) { fs.delete(index, false); } indexStream = fs.create(index); outStream = fs.create(masterIndex); String version = VERSION + " \n"; outStream.write(version.getBytes(Charsets.UTF_8)); } catch(IOException e) { throw new RuntimeException(e); } }
/** * Helper function to generate a {@link Path} for a file that is unique for * the task within the job output directory. * * <p>The path can be used to create custom files from within the map and * reduce tasks. The path name will be unique for each task. The path parent * will be the job output directory.</p>ls * * <p>This method uses the {@link #getUniqueName} method to make the file name * unique for the task.</p> * * @param conf the configuration for the job. * @param name the name for the file. * @return a unique path accross all tasks of the job. */ public static Path getPathForCustomFile(JobConf conf, String name) { return new Path(getWorkOutputPath(conf), getUniqueName(conf, name)); } }
/** * Helper function to generate a {@link Path} for a file that is unique for * the task within the job output directory. * * <p>The path can be used to create custom files from within the map and * reduce tasks. The path name will be unique for each task. The path parent * will be the job output directory.</p>ls * * <p>This method uses the {@link #getUniqueName} method to make the file name * unique for the task.</p> * * @param conf the configuration for the job. * @param name the name for the file. * @return a unique path accross all tasks of the job. */ public static Path getPathForCustomFile(JobConf conf, String name) { return new Path(getWorkOutputPath(conf), getUniqueName(conf, name)); } }
/** * Helper function to generate a {@link Path} for a file that is unique for * the task within the job output directory. * * <p>The path can be used to create custom files from within the map and * reduce tasks. The path name will be unique for each task. The path parent * will be the job output directory.</p>ls * * <p>This method uses the {@link #getUniqueName} method to make the file name * unique for the task.</p> * * @param conf the configuration for the job. * @param name the name for the file. * @return a unique path accross all tasks of the job. */ public static Path getPathForCustomFile(JobConf conf, String name) { return new Path(getWorkOutputPath(conf), getUniqueName(conf, name)); } }
/** * Helper function to generate a {@link Path} for a file that is unique for * the task within the job output directory. * * <p>The path can be used to create custom files from within the map and * reduce tasks. The path name will be unique for each task. The path parent * will be the job output directory.</p>ls * * <p>This method uses the {@link #getUniqueName} method to make the file name * unique for the task.</p> * * @param conf the configuration for the job. * @param name the name for the file. * @return a unique path accross all tasks of the job. */ public static Path getPathForCustomFile(JobConf conf, String name) { return new Path(getWorkOutputPath(conf), getUniqueName(conf, name)); } }
/** * Helper function to generate a {@link Path} for a file that is unique for * the task within the job output directory. * * <p>The path can be used to create custom files from within the map and * reduce tasks. The path name will be unique for each task. The path parent * will be the job output directory.</p>ls * * <p>This method uses the {@link #getUniqueName} method to make the file name * unique for the task.</p> * * @param conf the configuration for the job. * @param name the name for the file. * @return a unique path accross all tasks of the job. */ public static Path getPathForCustomFile(JobConf conf, String name) { return new Path(getWorkOutputPath(conf), getUniqueName(conf, name)); } }
public void configure(JobConf conf) { this.conf = conf; tmpOutputDir = FileOutputFormat.getWorkOutputPath(this.conf); masterIndex = new Path(tmpOutputDir, "_masterindex"); index = new Path(tmpOutputDir, "_index"); replication = conf.getInt(HAR_REPLICATION_LABEL, 3); try { fs = masterIndex.getFileSystem(conf); if (fs.exists(masterIndex)) { fs.delete(masterIndex, false); } if (fs.exists(index)) { fs.delete(index, false); } indexStream = fs.create(index); outStream = fs.create(masterIndex); String version = VERSION + " \n"; outStream.write(version.getBytes(Charsets.UTF_8)); } catch(IOException e) { throw new RuntimeException(e); } }
private Path getTaskAttemptPath(TaskAttemptContext context, Path out) throws IOException { Path workPath = FileOutputFormat.getWorkOutputPath(context.getJobConf()); if(workPath == null && out != null) { return org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter .getTaskAttemptPath(context, out); } return workPath; }
private Path getTaskAttemptPath(TaskAttemptContext context, Path out) throws IOException { Path workPath = FileOutputFormat.getWorkOutputPath(context.getJobConf()); if(workPath == null && out != null) { return org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter .getTaskAttemptPath(context, out); } return workPath; }
private Path getTaskAttemptPath(TaskAttemptContext context, Path out) throws IOException { Path workPath = FileOutputFormat.getWorkOutputPath(context.getJobConf()); if(workPath == null && out != null) { return org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter .getTaskAttemptPath(context, out); } return workPath; }
private Path getTaskAttemptPath(TaskAttemptContext context, Path out) throws IOException { Path workPath = FileOutputFormat.getWorkOutputPath(context.getJobConf()); if(workPath == null && out != null) { return org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter .getTaskAttemptPath(context, out); } return workPath; }
tmpOutputDir = FileOutputFormat.getWorkOutputPath(conf); blockSize = conf.getLong(HAR_BLOCKSIZE_LABEL, blockSize);
tmpOutputDir = FileOutputFormat.getWorkOutputPath(conf); blockSize = conf.getLong(HAR_BLOCKSIZE_LABEL, blockSize);
tmpOutputDir = FileOutputFormat.getWorkOutputPath(conf); blockSize = conf.getLong(HAR_BLOCKSIZE_LABEL, blockSize);