private static void spillIterator(UnsafeSorterIterator inMemIterator, UnsafeSorterSpillWriter spillWriter) throws IOException { while (inMemIterator.hasNext()) { inMemIterator.loadNext(); final Object baseObject = inMemIterator.getBaseObject(); final long baseOffset = inMemIterator.getBaseOffset(); final int recordLength = inMemIterator.getRecordLength(); spillWriter.write(baseObject, baseOffset, recordLength, inMemIterator.getKeyPrefix()); } spillWriter.close(); }
/** * Add an UnsafeSorterIterator to this merger */ public void addSpillIfNotEmpty(UnsafeSorterIterator spillReader) throws IOException { if (spillReader.hasNext()) { // We only add the spillReader to the priorityQueue if it is not empty. We do this to // make sure the hasNext method of UnsafeSorterIterator returned by getSortedIterator // does not return wrong result because hasNext will returns true // at least priorityQueue.size() times. If we allow n spillReaders in the // priorityQueue, we will have n extra empty records in the result of UnsafeSorterIterator. spillReader.loadNext(); priorityQueue.add(spillReader); numRecords += spillReader.getNumRecords(); } }
@Test public void testSortingEmptyArrays() throws Exception { final UnsafeExternalSorter sorter = newSorter(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(0, iter.getKeyPrefix()); assertEquals(0, iter.getRecordLength()); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Override public int compare(UnsafeSorterIterator left, UnsafeSorterIterator right) { final int prefixComparisonResult = prefixComparator.compare(left.getKeyPrefix(), right.getKeyPrefix()); if (prefixComparisonResult == 0) { return recordComparator.compare( left.getBaseObject(), left.getBaseOffset(), right.getBaseObject(), right.getBaseOffset()); } else { return prefixComparisonResult; } } };
@Override public int getRecordLength() { return upstream.getRecordLength(); }
@Override public long getBaseOffset() { return upstream.getBaseOffset(); }
@Override public Object getBaseObject() { return upstream.getBaseObject(); }
@Override public void loadNext() throws IOException { synchronized (this) { loaded = true; if (nextUpstream != null) { // Just consumed the last record from in memory iterator if (lastPage != null) { freePage(lastPage); lastPage = null; } upstream = nextUpstream; nextUpstream = null; } numRecords--; upstream.loadNext(); } }
@Override public long getKeyPrefix() { return upstream.getKeyPrefix(); } }
SpillableIterator(UnsafeSorterIterator inMemIterator) { this.upstream = inMemIterator; this.numRecords = inMemIterator.getNumRecords(); }
@Test public void testSortingEmptyArrays() throws Exception { final UnsafeExternalSorter sorter = newSorter(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(0, iter.getKeyPrefix()); assertEquals(0, iter.getRecordLength()); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Override public int getRecordLength() { return upstream.getRecordLength(); }
@Override public boolean hasNext() { return !priorityQueue.isEmpty() || (spillReader != null && spillReader.hasNext()); }
@Override public long getBaseOffset() { return upstream.getBaseOffset(); }
@Override public Object getBaseObject() { return upstream.getBaseObject(); }
@Override public void loadNext() throws IOException { synchronized (this) { loaded = true; if (nextUpstream != null) { // Just consumed the last record from in memory iterator if (lastPage != null) { freePage(lastPage); lastPage = null; } upstream = nextUpstream; nextUpstream = null; } numRecords--; upstream.loadNext(); } }
@Override public long getKeyPrefix() { return upstream.getKeyPrefix(); } }