@Override public boolean hasNext() { return !priorityQueue.isEmpty() || (spillReader != null && spillReader.hasNext()); }
@Override public boolean hasNext() { return !priorityQueue.isEmpty() || (spillReader != null && spillReader.hasNext()); }
@Override public boolean hasNext() { return !priorityQueue.isEmpty() || (spillReader != null && spillReader.hasNext()); }
@Override public void loadNext() throws IOException { while (!current.hasNext() && !iterators.isEmpty()) { current = iterators.remove(); } current.loadNext(); }
@Override public void loadNext() throws IOException { if (spillReader != null) { if (spillReader.hasNext()) { spillReader.loadNext(); priorityQueue.add(spillReader); } } spillReader = priorityQueue.remove(); }
@Override public void loadNext() throws IOException { while (!current.hasNext() && !iterators.isEmpty()) { current = iterators.remove(); } current.loadNext(); }
@Override public void loadNext() throws IOException { while (!current.hasNext() && !iterators.isEmpty()) { current = iterators.remove(); } current.loadNext(); }
@Override public void loadNext() throws IOException { if (spillReader != null) { if (spillReader.hasNext()) { spillReader.loadNext(); priorityQueue.add(spillReader); } } spillReader = priorityQueue.remove(); }
@Override public void loadNext() throws IOException { if (spillReader != null) { if (spillReader.hasNext()) { spillReader.loadNext(); priorityQueue.add(spillReader); } } spillReader = priorityQueue.remove(); }
private void moveOver(UnsafeSorterIterator iter, int steps) throws IOException { if (steps > 0) { for (int i = 0; i < steps; i++) { if (iter.hasNext()) { iter.loadNext(); } else { throw new ArrayIndexOutOfBoundsException("Failed to move the iterator " + steps + " steps forward"); } } } }
private void moveOver(UnsafeSorterIterator iter, int steps) throws IOException { if (steps > 0) { for (int i = 0; i < steps; i++) { if (iter.hasNext()) { iter.loadNext(); } else { throw new ArrayIndexOutOfBoundsException("Failed to move the iterator " + steps + " steps forward"); } } } }
private static void spillIterator(UnsafeSorterIterator inMemIterator, UnsafeSorterSpillWriter spillWriter) throws IOException { while (inMemIterator.hasNext()) { inMemIterator.loadNext(); final Object baseObject = inMemIterator.getBaseObject(); final long baseOffset = inMemIterator.getBaseOffset(); final int recordLength = inMemIterator.getRecordLength(); spillWriter.write(baseObject, baseOffset, recordLength, inMemIterator.getKeyPrefix()); } spillWriter.close(); }
private static void spillIterator(UnsafeSorterIterator inMemIterator, UnsafeSorterSpillWriter spillWriter) throws IOException { while (inMemIterator.hasNext()) { inMemIterator.loadNext(); final Object baseObject = inMemIterator.getBaseObject(); final long baseOffset = inMemIterator.getBaseOffset(); final int recordLength = inMemIterator.getRecordLength(); spillWriter.write(baseObject, baseOffset, recordLength, inMemIterator.getKeyPrefix()); } spillWriter.close(); }
@Test public void testSortingEmptyInput() { final TaskMemoryManager memoryManager = new TaskMemoryManager( new TestMemoryManager(new SparkConf().set("spark.memory.offHeap.enabled", "false")), 0); final TestMemoryConsumer consumer = new TestMemoryConsumer(memoryManager); final UnsafeInMemorySorter sorter = new UnsafeInMemorySorter(consumer, memoryManager, mock(RecordComparator.class), mock(PrefixComparator.class), 100, shouldUseRadixSort()); final UnsafeSorterIterator iter = sorter.getSortedIterator(); Assert.assertFalse(iter.hasNext()); }
@Test public void testSortingEmptyInput() { final TaskMemoryManager memoryManager = new TaskMemoryManager( new TestMemoryManager(new SparkConf().set("spark.memory.offHeap.enabled", "false")), 0); final TestMemoryConsumer consumer = new TestMemoryConsumer(memoryManager); final UnsafeInMemorySorter sorter = new UnsafeInMemorySorter(consumer, memoryManager, mock(RecordComparator.class), mock(PrefixComparator.class), 100, shouldUseRadixSort()); final UnsafeSorterIterator iter = sorter.getSortedIterator(); Assert.assertFalse(iter.hasNext()); }
private void verifyIntIterator(UnsafeSorterIterator iter, int start, int end) throws IOException { for (int i = start; i < end; i++) { assert (iter.hasNext()); iter.loadNext(); assert (Platform.getInt(iter.getBaseObject(), iter.getBaseOffset()) == i); } } }
private void verifyIntIterator(UnsafeSorterIterator iter, int start, int end) throws IOException { for (int i = start; i < end; i++) { assert (iter.hasNext()); iter.loadNext(); assert (Platform.getInt(iter.getBaseObject(), iter.getBaseOffset()) == i); } } }