@Override public int getRecordLength() { return upstream.getRecordLength(); }
@Override public int getRecordLength() { return upstream.getRecordLength(); }
@Override public int getRecordLength() { return upstream.getRecordLength(); }
@Override public int getRecordLength() { return current.getRecordLength(); }
@Override public int getRecordLength() { return current.getRecordLength(); }
@Override public int getRecordLength() { return spillReader.getRecordLength(); }
@Override public int getRecordLength() { return current.getRecordLength(); }
@Override public int getRecordLength() { return spillReader.getRecordLength(); }
@Override public int getRecordLength() { return spillReader.getRecordLength(); }
UnsafeSorterSpillMerger( RecordComparator recordComparator, PrefixComparator prefixComparator, int numSpills) { Comparator<UnsafeSorterIterator> comparator = (left, right) -> { int prefixComparisonResult = prefixComparator.compare(left.getKeyPrefix(), right.getKeyPrefix()); if (prefixComparisonResult == 0) { return recordComparator.compare( left.getBaseObject(), left.getBaseOffset(), left.getRecordLength(), right.getBaseObject(), right.getBaseOffset(), right.getRecordLength()); } else { return prefixComparisonResult; } }; priorityQueue = new PriorityQueue<>(numSpills, comparator); }
UnsafeSorterSpillMerger( RecordComparator recordComparator, PrefixComparator prefixComparator, int numSpills) { Comparator<UnsafeSorterIterator> comparator = (left, right) -> { int prefixComparisonResult = prefixComparator.compare(left.getKeyPrefix(), right.getKeyPrefix()); if (prefixComparisonResult == 0) { return recordComparator.compare( left.getBaseObject(), left.getBaseOffset(), left.getRecordLength(), right.getBaseObject(), right.getBaseOffset(), right.getRecordLength()); } else { return prefixComparisonResult; } }; priorityQueue = new PriorityQueue<>(numSpills, comparator); }
UnsafeSorterSpillMerger( RecordComparator recordComparator, PrefixComparator prefixComparator, int numSpills) { Comparator<UnsafeSorterIterator> comparator = (left, right) -> { int prefixComparisonResult = prefixComparator.compare(left.getKeyPrefix(), right.getKeyPrefix()); if (prefixComparisonResult == 0) { return recordComparator.compare( left.getBaseObject(), left.getBaseOffset(), left.getRecordLength(), right.getBaseObject(), right.getBaseOffset(), right.getRecordLength()); } else { return prefixComparisonResult; } }; priorityQueue = new PriorityQueue<>(numSpills, comparator); }
private static void spillIterator(UnsafeSorterIterator inMemIterator, UnsafeSorterSpillWriter spillWriter) throws IOException { while (inMemIterator.hasNext()) { inMemIterator.loadNext(); final Object baseObject = inMemIterator.getBaseObject(); final long baseOffset = inMemIterator.getBaseOffset(); final int recordLength = inMemIterator.getRecordLength(); spillWriter.write(baseObject, baseOffset, recordLength, inMemIterator.getKeyPrefix()); } spillWriter.close(); }
private static void spillIterator(UnsafeSorterIterator inMemIterator, UnsafeSorterSpillWriter spillWriter) throws IOException { while (inMemIterator.hasNext()) { inMemIterator.loadNext(); final Object baseObject = inMemIterator.getBaseObject(); final long baseOffset = inMemIterator.getBaseOffset(); final int recordLength = inMemIterator.getRecordLength(); spillWriter.write(baseObject, baseOffset, recordLength, inMemIterator.getKeyPrefix()); } spillWriter.close(); }
@Test public void testSortingEmptyArrays() throws Exception { final UnsafeExternalSorter sorter = newSorter(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(0, iter.getKeyPrefix()); assertEquals(0, iter.getRecordLength()); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingEmptyArrays() throws Exception { final UnsafeExternalSorter sorter = newSorter(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(0, iter.getKeyPrefix()); assertEquals(0, iter.getRecordLength()); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingEmptyArrays() throws Exception { final UnsafeExternalSorter sorter = newSorter(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(0, iter.getKeyPrefix()); assertEquals(0, iter.getRecordLength()); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingOnlyByPrefix() throws Exception { final UnsafeExternalSorter sorter = newSorter(); insertNumber(sorter, 5); insertNumber(sorter, 1); insertNumber(sorter, 3); sorter.spill(); insertNumber(sorter, 4); sorter.spill(); insertNumber(sorter, 2); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingOnlyByPrefix() throws Exception { final UnsafeExternalSorter sorter = newSorter(); insertNumber(sorter, 5); insertNumber(sorter, 1); insertNumber(sorter, 3); sorter.spill(); insertNumber(sorter, 4); sorter.spill(); insertNumber(sorter, 2); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingOnlyByPrefix() throws Exception { final UnsafeExternalSorter sorter = newSorter(); insertNumber(sorter, 5); insertNumber(sorter, 1); insertNumber(sorter, 3); sorter.spill(); insertNumber(sorter, 4); sorter.spill(); insertNumber(sorter, 2); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }