Arrays.sort(dataToSort); for (int i = 0; i < dataToSort.length; i++) { Assert.assertTrue(iter.hasNext()); iter.loadNext(); final int partitionId = iter.packedRecordPointer.getPartitionId(); Assert.assertTrue(partitionId >= 0 && partitionId <= 3); Assert.assertTrue(Arrays.binarySearch(dataToSort, str) != -1); Assert.assertFalse(iter.hasNext());
Arrays.sort(dataToSort); for (int i = 0; i < dataToSort.length; i++) { Assert.assertTrue(iter.hasNext()); iter.loadNext(); final int partitionId = iter.packedRecordPointer.getPartitionId(); Assert.assertTrue(partitionId >= 0 && partitionId <= 3); Assert.assertTrue(Arrays.binarySearch(dataToSort, str) != -1); Assert.assertFalse(iter.hasNext());
Arrays.sort(dataToSort); for (int i = 0; i < dataToSort.length; i++) { Assert.assertTrue(iter.hasNext()); iter.loadNext(); final int partitionId = iter.packedRecordPointer.getPartitionId(); Assert.assertTrue(partitionId >= 0 && partitionId <= 3); Assert.assertTrue(Arrays.binarySearch(dataToSort, str) != -1); Assert.assertFalse(iter.hasNext());
@Test public void testSortingManyNumbers() throws Exception { ShuffleInMemorySorter sorter = new ShuffleInMemorySorter(consumer, 4, shouldUseRadixSort()); int[] numbersToSort = new int[128000]; Random random = new Random(16); for (int i = 0; i < numbersToSort.length; i++) { if (!sorter.hasSpaceForAnotherRecord()) { sorter.expandPointerArray(consumer.allocateArray(sorter.getMemoryUsage() / 8 * 2)); } numbersToSort[i] = random.nextInt(PackedRecordPointer.MAXIMUM_PARTITION_ID + 1); sorter.insertRecord(0, numbersToSort[i]); } Arrays.sort(numbersToSort); int[] sorterResult = new int[numbersToSort.length]; ShuffleInMemorySorter.ShuffleSorterIterator iter = sorter.getSortedIterator(); int j = 0; while (iter.hasNext()) { iter.loadNext(); sorterResult[j] = iter.packedRecordPointer.getPartitionId(); j += 1; } Assert.assertArrayEquals(numbersToSort, sorterResult); } }
/** * Return an iterator over record pointers in sorted order. */ public ShuffleSorterIterator getSortedIterator() { int offset = 0; if (useRadixSort) { offset = RadixSort.sort( array, pos, PackedRecordPointer.PARTITION_ID_START_BYTE_INDEX, PackedRecordPointer.PARTITION_ID_END_BYTE_INDEX, false, false); } else { MemoryBlock unused = new MemoryBlock( array.getBaseObject(), array.getBaseOffset() + pos * 8L, (array.size() - pos) * 8L); LongArray buffer = new LongArray(unused); Sorter<PackedRecordPointer, LongArray> sorter = new Sorter<>(new ShuffleSortDataFormat(buffer)); sorter.sort(array, 0, pos, SORT_COMPARATOR); } return new ShuffleSorterIterator(pos, array, offset); } }
@Test public void testSortingManyNumbers() throws Exception { ShuffleInMemorySorter sorter = new ShuffleInMemorySorter(consumer, 4, shouldUseRadixSort()); int[] numbersToSort = new int[128000]; Random random = new Random(16); for (int i = 0; i < numbersToSort.length; i++) { if (!sorter.hasSpaceForAnotherRecord()) { sorter.expandPointerArray(consumer.allocateArray(sorter.getMemoryUsage() / 8 * 2)); } numbersToSort[i] = random.nextInt(PackedRecordPointer.MAXIMUM_PARTITION_ID + 1); sorter.insertRecord(0, numbersToSort[i]); } Arrays.sort(numbersToSort); int[] sorterResult = new int[numbersToSort.length]; ShuffleInMemorySorter.ShuffleSorterIterator iter = sorter.getSortedIterator(); int j = 0; while (iter.hasNext()) { iter.loadNext(); sorterResult[j] = iter.packedRecordPointer.getPartitionId(); j += 1; } Assert.assertArrayEquals(numbersToSort, sorterResult); } }
@Test public void testSortingManyNumbers() throws Exception { ShuffleInMemorySorter sorter = new ShuffleInMemorySorter(consumer, 4, shouldUseRadixSort()); int[] numbersToSort = new int[128000]; Random random = new Random(16); for (int i = 0; i < numbersToSort.length; i++) { if (!sorter.hasSpaceForAnotherRecord()) { sorter.expandPointerArray(consumer.allocateArray(sorter.getMemoryUsage() / 8 * 2)); } numbersToSort[i] = random.nextInt(PackedRecordPointer.MAXIMUM_PARTITION_ID + 1); sorter.insertRecord(0, numbersToSort[i]); } Arrays.sort(numbersToSort); int[] sorterResult = new int[numbersToSort.length]; ShuffleInMemorySorter.ShuffleSorterIterator iter = sorter.getSortedIterator(); int j = 0; while (iter.hasNext()) { iter.loadNext(); sorterResult[j] = iter.packedRecordPointer.getPartitionId(); j += 1; } Assert.assertArrayEquals(numbersToSort, sorterResult); } }
/** * Return an iterator over record pointers in sorted order. */ public ShuffleSorterIterator getSortedIterator() { int offset = 0; if (useRadixSort) { offset = RadixSort.sort( array, pos, PackedRecordPointer.PARTITION_ID_START_BYTE_INDEX, PackedRecordPointer.PARTITION_ID_END_BYTE_INDEX, false, false); } else { MemoryBlock unused = new MemoryBlock( array.getBaseObject(), array.getBaseOffset() + pos * 8L, (array.size() - pos) * 8L); LongArray buffer = new LongArray(unused); Sorter<PackedRecordPointer, LongArray> sorter = new Sorter<>(new ShuffleSortDataFormat(buffer)); sorter.sort(array, 0, pos, SORT_COMPARATOR); } return new ShuffleSorterIterator(pos, array, offset); } }
/** * Return an iterator over record pointers in sorted order. */ public ShuffleSorterIterator getSortedIterator() { int offset = 0; if (useRadixSort) { offset = RadixSort.sort( array, pos, PackedRecordPointer.PARTITION_ID_START_BYTE_INDEX, PackedRecordPointer.PARTITION_ID_END_BYTE_INDEX, false, false); } else { MemoryBlock unused = new MemoryBlock( array.getBaseObject(), array.getBaseOffset() + pos * 8L, (array.size() - pos) * 8L); LongArray buffer = new LongArray(unused); Sorter<PackedRecordPointer, LongArray> sorter = new Sorter<>(new ShuffleSortDataFormat(buffer)); sorter.sort(array, 0, pos, SORT_COMPARATOR); } return new ShuffleSorterIterator(pos, array, offset); } }
@Test public void testSortingEmptyInput() { final ShuffleInMemorySorter sorter = new ShuffleInMemorySorter( consumer, 100, shouldUseRadixSort()); final ShuffleInMemorySorter.ShuffleSorterIterator iter = sorter.getSortedIterator(); Assert.assertFalse(iter.hasNext()); }
@Test public void testSortingEmptyInput() { final ShuffleInMemorySorter sorter = new ShuffleInMemorySorter( consumer, 100, shouldUseRadixSort()); final ShuffleInMemorySorter.ShuffleSorterIterator iter = sorter.getSortedIterator(); Assert.assertFalse(iter.hasNext()); }
@Test public void testSortingEmptyInput() { final ShuffleInMemorySorter sorter = new ShuffleInMemorySorter( consumer, 100, shouldUseRadixSort()); final ShuffleInMemorySorter.ShuffleSorterIterator iter = sorter.getSortedIterator(); Assert.assertFalse(iter.hasNext()); }