@Test public void testFillingPage() throws Exception { final UnsafeExternalSorter sorter = newSorter(); byte[] record = new byte[16]; while (sorter.getNumberOfAllocatedPages() < 2) { sorter.insertRecord(record, Platform.BYTE_ARRAY_OFFSET, record.length, 0, false); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingOnlyByPrefix() throws Exception { final UnsafeExternalSorter sorter = newSorter(); insertNumber(sorter, 5); insertNumber(sorter, 1); insertNumber(sorter, 3); sorter.spill(); insertNumber(sorter, 4); sorter.spill(); insertNumber(sorter, 2); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
private UnsafeExternalSorter newSorter() throws IOException { return UnsafeExternalSorter.create( taskMemoryManager, blockManager, serializerManager, taskContext, () -> recordComparator, prefixComparator, /* initialSize */ 1024, pageSizeBytes, spillThreshold, shouldUseRadixSort()); }
@Test public void testGetIterator() throws Exception { final UnsafeExternalSorter sorter = newSorter(); for (int i = 0; i < 100; i++) { insertNumber(sorter, i); } verifyIntIterator(sorter.getIterator(0), 0, 100); verifyIntIterator(sorter.getIterator(79), 79, 100); sorter.spill(); for (int i = 100; i < 200; i++) { insertNumber(sorter, i); } sorter.spill(); verifyIntIterator(sorter.getIterator(79), 79, 200); for (int i = 200; i < 300; i++) { insertNumber(sorter, i); } verifyIntIterator(sorter.getIterator(79), 79, 300); verifyIntIterator(sorter.getIterator(139), 139, 300); verifyIntIterator(sorter.getIterator(279), 279, 300); }
pageSizeBytes, UnsafeExternalSorter.DEFAULT_NUM_ELEMENTS_FOR_SPILL_THRESHOLD, shouldUseRadixSort()); try { for (int i = 0; i < numRecordsPerPage * 10; i++) { insertNumber(sorter, i); newPeakMemory = sorter.getPeakMemoryUsedBytes(); if (i % numRecordsPerPage == 0) { assertEquals(previousPeakMemory, newPeakMemory); for (int i = 0; i < numRecordsPerPage; i++) { insertNumber(sorter, i); } finally { sorter.cleanupResources(); assertSpillFilesWereCleanedUp();
@Test public void sortingRecordsThatExceedPageSize() throws Exception { final UnsafeExternalSorter sorter = newSorter(); final int[] largeRecord = new int[(int) pageSizeBytes + 16]; Arrays.fill(largeRecord, 456); Arrays.fill(smallRecord, 123); insertRecord(sorter, largeRecord, 456); sorter.spill(); insertRecord(sorter, smallRecord, 123); sorter.spill(); insertRecord(sorter, smallRecord, 123); insertRecord(sorter, largeRecord, 456); assertSpillFilesWereCleanedUp();
pageSizeBytes, spillThreshold, shouldUseRadixSort()); long[] record = new long[100]; int recordSize = record.length * 8; assertSpillFilesWereCleanedUp();
@Test public void testOOMDuringSpill() throws Exception { final UnsafeExternalSorter sorter = newSorter(); insertNumber(sorter, i); insertNumber(sorter, 1024); fail("expected OutOfMmoryError but it seems operation surprisingly succeeded");
@Test public void testSortTimeMetric() throws Exception { final UnsafeExternalSorter sorter = newSorter(); long prevSortTime = sorter.getSortTimeNanos(); assertEquals(prevSortTime, 0); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); assertThat(sorter.getSortTimeNanos(), greaterThan(prevSortTime)); prevSortTime = sorter.getSortTimeNanos(); sorter.spill(); // no sort needed assertEquals(sorter.getSortTimeNanos(), prevSortTime); sorter.insertRecord(null, 0, 0, 0, false); UnsafeSorterIterator iter = sorter.getSortedIterator(); assertThat(sorter.getSortTimeNanos(), greaterThan(prevSortTime)); }
pageSizeBytes, spillThreshold, shouldUseRadixSort()); try { for (int i = 0; i < numRecordsPerPage * 10; i++) { insertNumber(sorter, i); newPeakMemory = sorter.getPeakMemoryUsedBytes(); if (i % numRecordsPerPage == 0) { assertEquals(previousPeakMemory, newPeakMemory); for (int i = 0; i < numRecordsPerPage; i++) { insertNumber(sorter, i); } finally { sorter.cleanupResources(); assertSpillFilesWereCleanedUp();
@Test public void sortingRecordsThatExceedPageSize() throws Exception { final UnsafeExternalSorter sorter = newSorter(); final int[] largeRecord = new int[(int) pageSizeBytes + 16]; Arrays.fill(largeRecord, 456); Arrays.fill(smallRecord, 123); insertRecord(sorter, largeRecord, 456); sorter.spill(); insertRecord(sorter, smallRecord, 123); sorter.spill(); insertRecord(sorter, smallRecord, 123); insertRecord(sorter, largeRecord, 456); assertSpillFilesWereCleanedUp();
@Test public void testGetIterator() throws Exception { final UnsafeExternalSorter sorter = newSorter(); for (int i = 0; i < 100; i++) { insertNumber(sorter, i); } verifyIntIterator(sorter.getIterator(0), 0, 100); verifyIntIterator(sorter.getIterator(79), 79, 100); sorter.spill(); for (int i = 100; i < 200; i++) { insertNumber(sorter, i); } sorter.spill(); verifyIntIterator(sorter.getIterator(79), 79, 200); for (int i = 200; i < 300; i++) { insertNumber(sorter, i); } verifyIntIterator(sorter.getIterator(79), 79, 300); verifyIntIterator(sorter.getIterator(139), 139, 300); verifyIntIterator(sorter.getIterator(279), 279, 300); }
pageSizeBytes, spillThreshold, shouldUseRadixSort()); long[] record = new long[100]; int recordSize = record.length * 8; assertSpillFilesWereCleanedUp();
@Test public void testOOMDuringSpill() throws Exception { final UnsafeExternalSorter sorter = newSorter(); insertNumber(sorter, i); insertNumber(sorter, 1024); fail("expected OutOfMmoryError but it seems operation surprisingly succeeded");
@Test public void testSortTimeMetric() throws Exception { final UnsafeExternalSorter sorter = newSorter(); long prevSortTime = sorter.getSortTimeNanos(); assertEquals(prevSortTime, 0); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); assertThat(sorter.getSortTimeNanos(), greaterThan(prevSortTime)); prevSortTime = sorter.getSortTimeNanos(); sorter.spill(); // no sort needed assertEquals(sorter.getSortTimeNanos(), prevSortTime); sorter.insertRecord(null, 0, 0, 0, false); UnsafeSorterIterator iter = sorter.getSortedIterator(); assertThat(sorter.getSortTimeNanos(), greaterThan(prevSortTime)); }
@Test public void testFillingPage() throws Exception { final UnsafeExternalSorter sorter = newSorter(); byte[] record = new byte[16]; while (sorter.getNumberOfAllocatedPages() < 2) { sorter.insertRecord(record, Platform.BYTE_ARRAY_OFFSET, record.length, 0, false); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingOnlyByPrefix() throws Exception { final UnsafeExternalSorter sorter = newSorter(); insertNumber(sorter, 5); insertNumber(sorter, 1); insertNumber(sorter, 3); sorter.spill(); insertNumber(sorter, 4); sorter.spill(); insertNumber(sorter, 2); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
pageSizeBytes, spillThreshold, shouldUseRadixSort()); try { for (int i = 0; i < numRecordsPerPage * 10; i++) { insertNumber(sorter, i); newPeakMemory = sorter.getPeakMemoryUsedBytes(); if (i % numRecordsPerPage == 0) { assertEquals(previousPeakMemory, newPeakMemory); for (int i = 0; i < numRecordsPerPage; i++) { insertNumber(sorter, i); } finally { sorter.cleanupResources(); assertSpillFilesWereCleanedUp();
@Test public void sortingRecordsThatExceedPageSize() throws Exception { final UnsafeExternalSorter sorter = newSorter(); final int[] largeRecord = new int[(int) pageSizeBytes + 16]; Arrays.fill(largeRecord, 456); Arrays.fill(smallRecord, 123); insertRecord(sorter, largeRecord, 456); sorter.spill(); insertRecord(sorter, smallRecord, 123); sorter.spill(); insertRecord(sorter, smallRecord, 123); insertRecord(sorter, largeRecord, 456); assertSpillFilesWereCleanedUp();
pageSizeBytes, UnsafeExternalSorter.DEFAULT_NUM_ELEMENTS_FOR_SPILL_THRESHOLD, shouldUseRadixSort()); long[] record = new long[100]; int recordSize = record.length * 8; assertSpillFilesWereCleanedUp();