segment.init(readsCounter); long startPos = segment.getReader().bytesRead; boolean hasNext = segment.nextRawKey(); long endPos = segment.getReader().bytesRead; segment.close(); totalBytes = 0; for (int i = 0; i < segmentsToMerge.size(); i++) { totalBytes += segmentsToMerge.get(i).getRawDataLength(); approxOutputSize += s.getLength() + ChecksumFileSystem.getApproxChkSumLength( s.getLength()); new Segment<K, V>(conf, fs, outputFile, codec, false); totalBytes -= inputBytesOfThisMerge - tempSegment.getRawDataLength(); if (totalBytes != 0) { progPerByte = 1.0f / (float)totalBytes;
if (s.inMemory()) { int offset = (i == 0) ? firstSegmentOffset : 0; s.getReader().reset(offset); } else { s.closeReader(); if (i == 0) { s.reinitReader(firstSegmentOffset); s.getReader().disableChecksumValidation();
segment.init(readsCounter); long startPos = segment.getReader().bytesRead; boolean hasNext = segment.nextRawKey(); long endPos = segment.getReader().bytesRead; segment.close(); totalBytes = 0; for (int i = 0; i < segmentsToMerge.size(); i++) { totalBytes += segmentsToMerge.get(i).getRawDataLength(); approxOutputSize += s.getLength() + ChecksumFileSystem.getApproxChkSumLength( s.getLength()); new Segment<K, V>(conf, fs, outputFile, codec, false); totalBytes -= inputBytesOfThisMerge - tempSegment.getRawDataLength(); if (totalBytes != 0) { progPerByte = 1.0f / (float)totalBytes;
segment.init(readsCounter); long startPos = segment.getPosition(); boolean hasNext = segment.nextRawKey(); long endPos = segment.getPosition(); segment.close(); totalBytes = 0; for (int i = 0; i < segmentsToMerge.size(); i++) { totalBytes += segmentsToMerge.get(i).getLength(); approxOutputSize += s.getLength() + ChecksumFileSystem.getApproxChkSumLength( s.getLength()); new Segment<K, V>(conf, fs, outputFile, codec, false); totalBytes -= inputBytesOfThisMerge - tempSegment.getLength(); if (totalBytes != 0) { progPerByte = 1.0f / (float)totalBytes;
segment.init(readsCounter); long startPos = segment.getReader().bytesRead; boolean hasNext = segment.nextRawKey(); long endPos = segment.getReader().bytesRead; segment.close(); totalBytes = 0; for (int i = 0; i < segmentsToMerge.size(); i++) { totalBytes += segmentsToMerge.get(i).getRawDataLength(); approxOutputSize += s.getLength() + ChecksumFileSystem.getApproxChkSumLength( s.getLength()); new Segment<K, V>(conf, fs, outputFile, codec, false); totalBytes -= inputBytesOfThisMerge - tempSegment.getRawDataLength(); if (totalBytes != 0) { progPerByte = 1.0f / (float)totalBytes;
segment.init(readsCounter); long startPos = segment.getReader().bytesRead; boolean hasNext = segment.nextRawKey(); long endPos = segment.getReader().bytesRead; segment.close(); totalBytes = 0; for (int i = 0; i < segmentsToMerge.size(); i++) { totalBytes += segmentsToMerge.get(i).getRawDataLength(); approxOutputSize += s.getLength() + ChecksumFileSystem.getApproxChkSumLength( s.getLength()); new Segment<K, V>(conf, fs, outputFile, codec, false); totalBytes -= inputBytesOfThisMerge - tempSegment.getRawDataLength(); if (totalBytes != 0) { progPerByte = 1.0f / (float)totalBytes;
new Segment<K, V>(job, rfs, filename[i], segmentOffset, segmentLength, codec, true); segmentList.add(i, s);
new Segment<K,V>(job, rfs, filename[i], indexRecord.startOffset, indexRecord.partLength, codec, true); segmentList.add(i, s);
new Segment<K,V>(job, rfs, filename[i], indexRecord.startOffset, indexRecord.partLength, codec, true); segmentList.add(i, s);
long startPos = segment.getPosition(); boolean hasNext = segment.next(); long endPos = segment.getPosition(); startBytes += endPos - startPos; segment.close(); totalBytesExpected += segmentsToMerge.get(i).getLength(); approxOutputSize += s.getLength() + ChecksumFileSystem.getApproxChkSumLength( s.getLength()); new Segment<K, V>(conf, fs, outputFile, codec, false); segments.add(tempSegment); numSegments = segments.size();
long startPos = segment.getPosition(); boolean hasNext = segment.next(); long endPos = segment.getPosition(); totalBytesProcessed += endPos - startPos; mergeProgress.set(totalBytesProcessed * progPerByte); segment.close(); totalBytes += segmentsToMerge.get(i).getLength(); approxOutputSize += s.getLength() + ChecksumFileSystem.getApproxChkSumLength( s.getLength()); new Segment<K, V>(conf, fs, outputFile, codec, false); segments.add(tempSegment); numSegments = segments.size();
nextKVOffset = (int) seg.getActualPosition(); if (seg.nextRawKey()) { currentKey = seg.getKey(); seg.getValue(currentValue); hasMore = true; return true; } else { if (!seg.inMemory()) { seg.closeReader(); if (!nextSegment.inMemory()) { currentValue.reset(currentDiskValue.getData(), currentDiskValue.getLength()); nextSegment.init(null); if (nextSegment.nextRawKey()) { currentKey = nextSegment.getKey(); nextSegment.getValue(currentValue); hasMore = true; return true;
nextKVOffset = (int) seg.getActualPosition(); if (seg.nextRawKey()) { currentKey = seg.getKey(); seg.getValue(currentValue); hasMore = true; return true; } else { if (!seg.inMemory()) { seg.closeReader(); if (!nextSegment.inMemory()) { currentValue.reset(currentDiskValue.getData(), currentDiskValue.getLength()); nextSegment.init(null); if (nextSegment.nextRawKey()) { currentKey = nextSegment.getKey(); nextSegment.getValue(currentValue); hasMore = true; return true;
nextKVOffset = (int) seg.getActualPosition(); if (seg.nextRawKey()) { currentKey = seg.getKey(); seg.getValue(currentValue); hasMore = true; return true; } else { if (!seg.inMemory()) { seg.closeReader(); if (!nextSegment.inMemory()) { currentValue.reset(currentDiskValue.getData(), currentDiskValue.getLength()); nextSegment.init(null); if (nextSegment.nextRawKey()) { currentKey = nextSegment.getKey(); nextSegment.getValue(currentValue); hasMore = true; return true;
nextKVOffset = (int) seg.getActualPosition(); if (seg.nextRawKey()) { currentKey = seg.getKey(); seg.getValue(currentValue); hasMore = true; return true; } else { if (!seg.inMemory()) { seg.closeReader(); if (!nextSegment.inMemory()) { currentValue.reset(currentDiskValue.getData(), currentDiskValue.getLength()); nextSegment.init(null); if (nextSegment.nextRawKey()) { currentKey = nextSegment.getKey(); nextSegment.getValue(currentValue); hasMore = true; return true;
nextKVOffset = (int) seg.getActualPosition(); if (seg.nextRawKey()) { currentKey = seg.getKey(); seg.getValue(currentValue); hasMore = true; return true; } else { if (!seg.inMemory()) { seg.closeReader(); if (!nextSegment.inMemory()) { currentValue.reset(currentDiskValue.getData(), currentDiskValue.getLength()); nextSegment.init(null); if (nextSegment.nextRawKey()) { currentKey = nextSegment.getKey(); nextSegment.getValue(currentValue); hasMore = true; return true;
long startPos = minSegment.getReader().bytesRead; key = minSegment.getKey(); if (!minSegment.inMemory()) { minSegment.getValue(diskIFileValue); value.reset(diskIFileValue.getData(), diskIFileValue.getLength()); } else { minSegment.getValue(value); long endPos = minSegment.getReader().bytesRead; totalBytesProcessed += endPos - startPos; mergeProgress.set(totalBytesProcessed * progPerByte);
long startPos = minSegment.getReader().bytesRead; key = minSegment.getKey(); if (!minSegment.inMemory()) { minSegment.getValue(diskIFileValue); value.reset(diskIFileValue.getData(), diskIFileValue.getLength()); } else { minSegment.getValue(value); long endPos = minSegment.getReader().bytesRead; totalBytesProcessed += endPos - startPos; mergeProgress.set(totalBytesProcessed * progPerByte);
long startPos = minSegment.getReader().bytesRead; key = minSegment.getKey(); if (!minSegment.inMemory()) { minSegment.getValue(diskIFileValue); value.reset(diskIFileValue.getData(), diskIFileValue.getLength()); } else { minSegment.getValue(value); long endPos = minSegment.getReader().bytesRead; totalBytesProcessed += endPos - startPos; mergeProgress.set(Math.min(1.0f, totalBytesProcessed * progPerByte));
long startPos = minSegment.getReader().bytesRead; key = minSegment.getKey(); if (!minSegment.inMemory()) { minSegment.getValue(diskIFileValue); value.reset(diskIFileValue.getData(), diskIFileValue.getLength()); } else { minSegment.getValue(value); long endPos = minSegment.getReader().bytesRead; totalBytesProcessed += endPos - startPos; mergeProgress.set(totalBytesProcessed * progPerByte);