/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param tempDir the directory for creating temp files during merge * @param deleteInputs true if the input files should be deleted when * unnecessary * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, Path tempDir, boolean deleteInputs) throws IOException { //outFile will basically be used as prefix for temp files for the //intermediate merge outputs this.outFile = new Path(tempDir + Path.SEPARATOR + "merged"); //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getFileStatus(inNames[i]).getLen(), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } factor = (inNames.length < factor) ? inNames.length : factor; // pass in object to report progress, if present MergeQueue mQueue = new MergeQueue(a, tempDir, progressable); return mQueue.merge(); }
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param deleteInputs true if the input files should be deleted when * unnecessary * @param factor the factor that will be used as the maximum merge fan-in * @param tmpDir the directory to write temporary files into * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, boolean deleteInputs, int factor, Path tmpDir) throws IOException { //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getFileStatus(inNames[i]).getLen(), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } this.factor = factor; MergeQueue mQueue = new MergeQueue(a, tmpDir, progressable); return mQueue.merge(); }
new SegmentDescriptor(0, fs.getFileStatus(outputFile).getLen(), outputFile);
new SegmentDescriptor(0, fs.getLength(outputFile), outputFile);
new SegmentDescriptor(0, fs.getFileStatus(outputFile).getLen(), outputFile);
new SegmentDescriptor(0, fs.getFileStatus(outputFile).getLen(), outputFile);
new SegmentDescriptor(0, fs.getFileStatus(outputFile).getLen(), outputFile);
new SegmentDescriptor(0, fs.getLength(outputFile), outputFile);
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param tempDir the directory for creating temp files during merge * @param deleteInputs true if the input files should be deleted when * unnecessary * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, Path tempDir, boolean deleteInputs) throws IOException { //outFile will basically be used as prefix for temp files for the //intermediate merge outputs this.outFile = new Path(tempDir + Path.SEPARATOR + "merged"); //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getLength(inNames[i]), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } factor = (inNames.length < factor) ? inNames.length : factor; // pass in object to report progress, if present MergeQueue mQueue = new MergeQueue(a, tempDir, progressable); return mQueue.merge(); }
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param tempDir the directory for creating temp files during merge * @param deleteInputs true if the input files should be deleted when * unnecessary * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, Path tempDir, boolean deleteInputs) throws IOException { //outFile will basically be used as prefix for temp files for the //intermediate merge outputs this.outFile = new Path(tempDir + Path.SEPARATOR + "merged"); //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getFileStatus(inNames[i]).getLen(), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } factor = (inNames.length < factor) ? inNames.length : factor; // pass in object to report progress, if present MergeQueue mQueue = new MergeQueue(a, tempDir, progressable); return mQueue.merge(); }
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param tempDir the directory for creating temp files during merge * @param deleteInputs true if the input files should be deleted when * unnecessary * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, Path tempDir, boolean deleteInputs) throws IOException { //outFile will basically be used as prefix for temp files for the //intermediate merge outputs this.outFile = new Path(tempDir + Path.SEPARATOR + "merged"); //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getFileStatus(inNames[i]).getLen(), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } factor = (inNames.length < factor) ? inNames.length : factor; // pass in object to report progress, if present MergeQueue mQueue = new MergeQueue(a, tempDir, progressable); return mQueue.merge(); }
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param tempDir the directory for creating temp files during merge * @param deleteInputs true if the input files should be deleted when * unnecessary * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, Path tempDir, boolean deleteInputs) throws IOException { //outFile will basically be used as prefix for temp files for the //intermediate merge outputs this.outFile = new Path(tempDir + Path.SEPARATOR + "merged"); //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getFileStatus(inNames[i]).getLen(), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } factor = (inNames.length < factor) ? inNames.length : factor; // pass in object to report progress, if present MergeQueue mQueue = new MergeQueue(a, tempDir, progressable); return mQueue.merge(); }
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param tempDir the directory for creating temp files during merge * @param deleteInputs true if the input files should be deleted when * unnecessary * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, Path tempDir, boolean deleteInputs) throws IOException { //outFile will basically be used as prefix for temp files for the //intermediate merge outputs this.outFile = new Path(tempDir + Path.SEPARATOR + "merged"); //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getFileStatus(inNames[i]).getLen(), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } factor = (inNames.length < factor) ? inNames.length : factor; // pass in object to report progress, if present MergeQueue mQueue = new MergeQueue(a, tempDir, progressable); return mQueue.merge(); }
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param deleteInputs true if the input files should be deleted when * unnecessary * @param factor the factor that will be used as the maximum merge fan-in * @param tmpDir the directory to write temporary files into * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, boolean deleteInputs, int factor, Path tmpDir) throws IOException { //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getFileStatus(inNames[i]).getLen(), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } this.factor = factor; MergeQueue mQueue = new MergeQueue(a, tmpDir, progressable); return mQueue.merge(); }
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param deleteInputs true if the input files should be deleted when * unnecessary * @param factor the factor that will be used as the maximum merge fan-in * @param tmpDir the directory to write temporary files into * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, boolean deleteInputs, int factor, Path tmpDir) throws IOException { //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getLength(inNames[i]), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } this.factor = factor; MergeQueue mQueue = new MergeQueue(a, tmpDir, progressable); return mQueue.merge(); }
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param deleteInputs true if the input files should be deleted when * unnecessary * @param factor the factor that will be used as the maximum merge fan-in * @param tmpDir the directory to write temporary files into * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, boolean deleteInputs, int factor, Path tmpDir) throws IOException { //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getFileStatus(inNames[i]).getLen(), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } this.factor = factor; MergeQueue mQueue = new MergeQueue(a, tmpDir, progressable); return mQueue.merge(); }
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param deleteInputs true if the input files should be deleted when * unnecessary * @param factor the factor that will be used as the maximum merge fan-in * @param tmpDir the directory to write temporary files into * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, boolean deleteInputs, int factor, Path tmpDir) throws IOException { //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getLength(inNames[i]), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } this.factor = factor; MergeQueue mQueue = new MergeQueue(a, tmpDir, progressable); return mQueue.merge(); }
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param tempDir the directory for creating temp files during merge * @param deleteInputs true if the input files should be deleted when * unnecessary * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, Path tempDir, boolean deleteInputs) throws IOException { //outFile will basically be used as prefix for temp files for the //intermediate merge outputs this.outFile = new Path(tempDir + Path.SEPARATOR + "merged"); //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getLength(inNames[i]), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } factor = (inNames.length < factor) ? inNames.length : factor; // pass in object to report progress, if present MergeQueue mQueue = new MergeQueue(a, tempDir, progressable); return mQueue.merge(); }
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param deleteInputs true if the input files should be deleted when * unnecessary * @param factor the factor that will be used as the maximum merge fan-in * @param tmpDir the directory to write temporary files into * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, boolean deleteInputs, int factor, Path tmpDir) throws IOException { //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getFileStatus(inNames[i]).getLen(), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } this.factor = factor; MergeQueue mQueue = new MergeQueue(a, tmpDir, progressable); return mQueue.merge(); }
/** * Merges the contents of files passed in Path[] * @param inNames the array of path names * @param deleteInputs true if the input files should be deleted when * unnecessary * @param factor the factor that will be used as the maximum merge fan-in * @param tmpDir the directory to write temporary files into * @return RawKeyValueIteratorMergeQueue * @throws IOException */ public RawKeyValueIterator merge(Path [] inNames, boolean deleteInputs, int factor, Path tmpDir) throws IOException { //get the segments from inNames ArrayList <SegmentDescriptor> a = new ArrayList <SegmentDescriptor>(); for (int i = 0; i < inNames.length; i++) { SegmentDescriptor s = new SegmentDescriptor(0, fs.getFileStatus(inNames[i]).getLen(), inNames[i]); s.preserveInput(!deleteInputs); s.doSync(); a.add(s); } this.factor = factor; MergeQueue mQueue = new MergeQueue(a, tmpDir, progressable); return mQueue.merge(); }