/** * Get spill offsets for given partition. */ public IndexRecord getIndex(int partition) { final int pos = partition * MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH / 8; return new IndexRecord(entries.get(pos), entries.get(pos + 1), entries.get(pos + 2)); }
/** * This function returns the index information for the given mapId, Spill * number and reducer combination. Index Information is obtained * transparently from either the indexMap or the underlying indexFile * @param mapId * @param spillNum * @param reducer * @return * @throws IOException */ private IndexRecord getIndexInformation( TaskAttemptID mapId, int spillNum, int reducer) throws IOException { IndexRecord[] irArray = null; if (indexCacheList.size() > spillNum) { irArray = indexCacheList.get(spillNum); } else { Path indexFileName = mapOutputFile.getSpillIndexFile(mapId, spillNum); irArray = IndexRecord.readIndexFile(indexFileName, job); indexCacheList.add(spillNum,irArray); rfs.delete(indexFileName,false); } return irArray[reducer]; }
/** * Get spill offsets for given partition. */ public IndexRecord getIndex(int partition) { final int pos = partition * MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH / 8; return new IndexRecord(entries.get(pos), entries.get(pos + 1), entries.get(pos + 2)); }
IndexRecord[] tmp = null; try { tmp = IndexRecord.readIndexFile(indexFileName, conf); } catch (Throwable e) { tmp = new IndexRecord[0];
/** * Get spill offsets for given partition. */ public IndexRecord getIndex(int partition) { final int pos = partition * MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH / 8; return new IndexRecord(entries.get(pos), entries.get(pos + 1), entries.get(pos + 2)); }
/** * Get spill offsets for given partition. */ public IndexRecord getIndex(int partition) { final int pos = partition * MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH / 8; return new IndexRecord(entries.get(pos), entries.get(pos + 1), entries.get(pos + 2)); }
/** * Get spill offsets for given partition. */ public IndexRecord getIndex(int partition) { final int pos = partition * MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH / 8; return new IndexRecord(entries.get(pos), entries.get(pos + 1), entries.get(pos + 2)); }
/** * Get spill offsets for given partition. */ public IndexRecord getIndex(int partition) { final int pos = partition * MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH / 8; return new IndexRecord(entries.get(pos), entries.get(pos + 1), entries.get(pos + 2)); }
long partLength = wrapper.readLong(); indexRecordArray[i] = new IndexRecord(startOffset, rawLength, partLength);
IndexRecord rec = new IndexRecord(); SpillRecord sr = new SpillRecord(partitions); try { IndexRecord rec = new IndexRecord(); final SpillRecord spillRec = new SpillRecord(partitions); for (int parts = 0; parts < partitions; parts++) {
IndexRecord rec = new IndexRecord(); SpillRecord sr = new SpillRecord(partitions); try { IndexRecord rec = new IndexRecord(); final SpillRecord spillRec = new SpillRecord(partitions); for (int parts = 0; parts < partitions; parts++) {
IndexRecord rec = new IndexRecord(); SpillRecord sr = new SpillRecord(partitions); try { IndexRecord rec = new IndexRecord(); final SpillRecord spillRec = new SpillRecord(partitions); for (int parts = 0; parts < partitions; parts++) {
sorter.sort(MapOutputBuffer.this, mstart, mend, reporter); int spindex = mstart; final IndexRecord rec = new IndexRecord(); final InMemValBytes value = new InMemValBytes(); for (int i = 0; i < partitions; ++i) {
IndexRecord rec = new IndexRecord(); SpillRecord sr = new SpillRecord(partitions); try { IndexRecord rec = new IndexRecord(); final SpillRecord spillRec = new SpillRecord(partitions); for (int parts = 0; parts < partitions; parts++) {
irArray[i] = new IndexRecord(segmentStart, writer.getRawLength(), writer.getCompressedLength());
IndexRecord rec = new IndexRecord(); SpillRecord sr = new SpillRecord(partitions); try { IndexRecord rec = new IndexRecord(); final SpillRecord spillRec = new SpillRecord(partitions); for (int parts = 0; parts < partitions; parts++) {
IndexRecord rec = new IndexRecord(); SpillRecord sr = new SpillRecord(partitions); try { IndexRecord rec = new IndexRecord(); final SpillRecord spillRec = new SpillRecord(partitions); for (int parts = 0; parts < partitions; parts++) {
private void copyPartitions(Path mapOutputPath, Path indexPath) throws IOException { FileSystem localFs = FileSystem.getLocal(jobConf); FileSystem rfs = ((LocalFileSystem)localFs).getRaw(); FSDataOutputStream rawOutput = rfs.create(mapOutputPath, true, BUF_SIZE); SpillRecord spillRecord = new SpillRecord(numberOfPartitions); IndexRecord indexRecord = new IndexRecord(); for (int i = 0; i < numberOfPartitions; i++) { indexRecord.startOffset = rawOutput.getPos(); byte buffer[] = outStreams[i].toByteArray(); IFileOutputStream checksumOutput = new IFileOutputStream(rawOutput); checksumOutput.write(buffer); // Write checksum. checksumOutput.finish(); // Write index record indexRecord.rawLength = (long)buffer.length; indexRecord.partLength = rawOutput.getPos() - indexRecord.startOffset; spillRecord.putIndex(indexRecord, i); reporter.progress(); } rawOutput.close(); spillRecord.writeToFile(indexPath, jobConf); } }
sorter.sort(MapOutputBuffer.this, mstart, mend, reporter); int spindex = mstart; final IndexRecord rec = new IndexRecord(); final InMemValBytes value = new InMemValBytes(); for (int i = 0; i < partitions; ++i) {
IndexRecord rec = new IndexRecord(); for (int i = 0; i < partitions; ++i) { IFile.Writer<K, V> writer = null;