@Override public void loadNext() throws IOException { synchronized (this) { loaded = true; if (nextUpstream != null) { // Just consumed the last record from in memory iterator if (lastPage != null) { freePage(lastPage); lastPage = null; } upstream = nextUpstream; nextUpstream = null; } numRecords--; upstream.loadNext(); } }
@Override public void loadNext() throws IOException { synchronized (this) { loaded = true; if (nextUpstream != null) { // Just consumed the last record from in memory iterator if (lastPage != null) { freePage(lastPage); lastPage = null; } upstream = nextUpstream; nextUpstream = null; } numRecords--; upstream.loadNext(); } }
@Override public void loadNext() throws IOException { synchronized (this) { loaded = true; if (nextUpstream != null) { // Just consumed the last record from in memory iterator if (lastPage != null) { freePage(lastPage); lastPage = null; } upstream = nextUpstream; nextUpstream = null; } numRecords--; upstream.loadNext(); } }
@Override public void loadNext() throws IOException { while (!current.hasNext() && !iterators.isEmpty()) { current = iterators.remove(); } current.loadNext(); }
@Override public void loadNext() throws IOException { if (spillReader != null) { if (spillReader.hasNext()) { spillReader.loadNext(); priorityQueue.add(spillReader); } } spillReader = priorityQueue.remove(); }
private void moveOver(UnsafeSorterIterator iter, int steps) throws IOException { if (steps > 0) { for (int i = 0; i < steps; i++) { if (iter.hasNext()) { iter.loadNext(); } else { throw new ArrayIndexOutOfBoundsException("Failed to move the iterator " + steps + " steps forward"); } } } }
@Override public void loadNext() throws IOException { while (!current.hasNext() && !iterators.isEmpty()) { current = iterators.remove(); } current.loadNext(); }
@Override public void loadNext() throws IOException { while (!current.hasNext() && !iterators.isEmpty()) { current = iterators.remove(); } current.loadNext(); }
@Override public void loadNext() throws IOException { if (spillReader != null) { if (spillReader.hasNext()) { spillReader.loadNext(); priorityQueue.add(spillReader); } } spillReader = priorityQueue.remove(); }
@Override public void loadNext() throws IOException { if (spillReader != null) { if (spillReader.hasNext()) { spillReader.loadNext(); priorityQueue.add(spillReader); } } spillReader = priorityQueue.remove(); }
private void moveOver(UnsafeSorterIterator iter, int steps) throws IOException { if (steps > 0) { for (int i = 0; i < steps; i++) { if (iter.hasNext()) { iter.loadNext(); } else { throw new ArrayIndexOutOfBoundsException("Failed to move the iterator " + steps + " steps forward"); } } } }
/** * Add an UnsafeSorterIterator to this merger */ public void addSpillIfNotEmpty(UnsafeSorterIterator spillReader) throws IOException { if (spillReader.hasNext()) { // We only add the spillReader to the priorityQueue if it is not empty. We do this to // make sure the hasNext method of UnsafeSorterIterator returned by getSortedIterator // does not return wrong result because hasNext will returns true // at least priorityQueue.size() times. If we allow n spillReaders in the // priorityQueue, we will have n extra empty records in the result of UnsafeSorterIterator. spillReader.loadNext(); priorityQueue.add(spillReader); numRecords += spillReader.getNumRecords(); } }
/** * Add an UnsafeSorterIterator to this merger */ public void addSpillIfNotEmpty(UnsafeSorterIterator spillReader) throws IOException { if (spillReader.hasNext()) { // We only add the spillReader to the priorityQueue if it is not empty. We do this to // make sure the hasNext method of UnsafeSorterIterator returned by getSortedIterator // does not return wrong result because hasNext will return true // at least priorityQueue.size() times. If we allow n spillReaders in the // priorityQueue, we will have n extra empty records in the result of UnsafeSorterIterator. spillReader.loadNext(); priorityQueue.add(spillReader); numRecords += spillReader.getNumRecords(); } }
/** * Add an UnsafeSorterIterator to this merger */ public void addSpillIfNotEmpty(UnsafeSorterIterator spillReader) throws IOException { if (spillReader.hasNext()) { // We only add the spillReader to the priorityQueue if it is not empty. We do this to // make sure the hasNext method of UnsafeSorterIterator returned by getSortedIterator // does not return wrong result because hasNext will return true // at least priorityQueue.size() times. If we allow n spillReaders in the // priorityQueue, we will have n extra empty records in the result of UnsafeSorterIterator. spillReader.loadNext(); priorityQueue.add(spillReader); numRecords += spillReader.getNumRecords(); } }
private static void spillIterator(UnsafeSorterIterator inMemIterator, UnsafeSorterSpillWriter spillWriter) throws IOException { while (inMemIterator.hasNext()) { inMemIterator.loadNext(); final Object baseObject = inMemIterator.getBaseObject(); final long baseOffset = inMemIterator.getBaseOffset(); final int recordLength = inMemIterator.getRecordLength(); spillWriter.write(baseObject, baseOffset, recordLength, inMemIterator.getKeyPrefix()); } spillWriter.close(); }
private static void spillIterator(UnsafeSorterIterator inMemIterator, UnsafeSorterSpillWriter spillWriter) throws IOException { while (inMemIterator.hasNext()) { inMemIterator.loadNext(); final Object baseObject = inMemIterator.getBaseObject(); final long baseOffset = inMemIterator.getBaseOffset(); final int recordLength = inMemIterator.getRecordLength(); spillWriter.write(baseObject, baseOffset, recordLength, inMemIterator.getKeyPrefix()); } spillWriter.close(); }
private void verifyIntIterator(UnsafeSorterIterator iter, int start, int end) throws IOException { for (int i = start; i < end; i++) { assert (iter.hasNext()); iter.loadNext(); assert (Platform.getInt(iter.getBaseObject(), iter.getBaseOffset()) == i); } } }
private void verifyIntIterator(UnsafeSorterIterator iter, int start, int end) throws IOException { for (int i = start; i < end; i++) { assert (iter.hasNext()); iter.loadNext(); assert (Platform.getInt(iter.getBaseObject(), iter.getBaseOffset()) == i); } } }
@Test public void testSortingEmptyArrays() throws Exception { final UnsafeExternalSorter sorter = newSorter(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(0, iter.getKeyPrefix()); assertEquals(0, iter.getRecordLength()); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingOnlyByPrefix() throws Exception { final UnsafeExternalSorter sorter = newSorter(); insertNumber(sorter, 5); insertNumber(sorter, 1); insertNumber(sorter, 3); sorter.spill(); insertNumber(sorter, 4); sorter.spill(); insertNumber(sorter, 2); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }