@Override public int compare(PackedRecordPointer left, PackedRecordPointer right) { int leftId = left.getPartitionId(); int rightId = right.getPartitionId(); return leftId < rightId ? -1 : (leftId > rightId ? 1 : 0); } }
@Override public int compare(PackedRecordPointer left, PackedRecordPointer right) { int leftId = left.getPartitionId(); int rightId = right.getPartitionId(); return leftId < rightId ? -1 : (leftId > rightId ? 1 : 0); } }
@Override public int compare(PackedRecordPointer left, PackedRecordPointer right) { int leftId = left.getPartitionId(); int rightId = right.getPartitionId(); return leftId < rightId ? -1 : (leftId > rightId ? 1 : 0); } }
@Test public void partitionIdsGreaterThanMaximumPartitionIdWillOverflowOrTriggerError() { PackedRecordPointer packedPointer = new PackedRecordPointer(); try { // Pointers greater than the maximum partition ID will overflow or trigger an assertion error packedPointer.set(PackedRecordPointer.packPointer(0, MAXIMUM_PARTITION_ID + 1)); assertFalse(MAXIMUM_PARTITION_ID + 1 == packedPointer.getPartitionId()); } catch (AssertionError e ) { // pass } }
@Test public void maximumPartitionIdCanBeEncoded() { PackedRecordPointer packedPointer = new PackedRecordPointer(); packedPointer.set(PackedRecordPointer.packPointer(0, MAXIMUM_PARTITION_ID)); assertEquals(MAXIMUM_PARTITION_ID, packedPointer.getPartitionId()); }
@Test public void maximumPartitionIdCanBeEncoded() { PackedRecordPointer packedPointer = new PackedRecordPointer(); packedPointer.set(PackedRecordPointer.packPointer(0, MAXIMUM_PARTITION_ID)); assertEquals(MAXIMUM_PARTITION_ID, packedPointer.getPartitionId()); }
@Test public void maximumPartitionIdCanBeEncoded() { PackedRecordPointer packedPointer = new PackedRecordPointer(); packedPointer.set(PackedRecordPointer.packPointer(0, MAXIMUM_PARTITION_ID)); assertEquals(MAXIMUM_PARTITION_ID, packedPointer.getPartitionId()); }
@Test public void partitionIdsGreaterThanMaximumPartitionIdWillOverflowOrTriggerError() { PackedRecordPointer packedPointer = new PackedRecordPointer(); try { // Pointers greater than the maximum partition ID will overflow or trigger an assertion error packedPointer.set(PackedRecordPointer.packPointer(0, MAXIMUM_PARTITION_ID + 1)); assertFalse(MAXIMUM_PARTITION_ID + 1 == packedPointer.getPartitionId()); } catch (AssertionError e ) { // pass } }
@Test public void partitionIdsGreaterThanMaximumPartitionIdWillOverflowOrTriggerError() { PackedRecordPointer packedPointer = new PackedRecordPointer(); try { // Pointers greater than the maximum partition ID will overflow or trigger an assertion error packedPointer.set(PackedRecordPointer.packPointer(0, MAXIMUM_PARTITION_ID + 1)); assertFalse(MAXIMUM_PARTITION_ID + 1 == packedPointer.getPartitionId()); } catch (AssertionError e ) { // pass } }
@Test public void testSortingManyNumbers() throws Exception { ShuffleInMemorySorter sorter = new ShuffleInMemorySorter(consumer, 4, shouldUseRadixSort()); int[] numbersToSort = new int[128000]; Random random = new Random(16); for (int i = 0; i < numbersToSort.length; i++) { if (!sorter.hasSpaceForAnotherRecord()) { sorter.expandPointerArray(consumer.allocateArray(sorter.getMemoryUsage() / 8 * 2)); } numbersToSort[i] = random.nextInt(PackedRecordPointer.MAXIMUM_PARTITION_ID + 1); sorter.insertRecord(0, numbersToSort[i]); } Arrays.sort(numbersToSort); int[] sorterResult = new int[numbersToSort.length]; ShuffleInMemorySorter.ShuffleSorterIterator iter = sorter.getSortedIterator(); int j = 0; while (iter.hasNext()) { iter.loadNext(); sorterResult[j] = iter.packedRecordPointer.getPartitionId(); j += 1; } Assert.assertArrayEquals(numbersToSort, sorterResult); } }
@Test public void testSortingManyNumbers() throws Exception { ShuffleInMemorySorter sorter = new ShuffleInMemorySorter(consumer, 4, shouldUseRadixSort()); int[] numbersToSort = new int[128000]; Random random = new Random(16); for (int i = 0; i < numbersToSort.length; i++) { if (!sorter.hasSpaceForAnotherRecord()) { sorter.expandPointerArray(consumer.allocateArray(sorter.getMemoryUsage() / 8 * 2)); } numbersToSort[i] = random.nextInt(PackedRecordPointer.MAXIMUM_PARTITION_ID + 1); sorter.insertRecord(0, numbersToSort[i]); } Arrays.sort(numbersToSort); int[] sorterResult = new int[numbersToSort.length]; ShuffleInMemorySorter.ShuffleSorterIterator iter = sorter.getSortedIterator(); int j = 0; while (iter.hasNext()) { iter.loadNext(); sorterResult[j] = iter.packedRecordPointer.getPartitionId(); j += 1; } Assert.assertArrayEquals(numbersToSort, sorterResult); } }
@Test public void testSortingManyNumbers() throws Exception { ShuffleInMemorySorter sorter = new ShuffleInMemorySorter(consumer, 4, shouldUseRadixSort()); int[] numbersToSort = new int[128000]; Random random = new Random(16); for (int i = 0; i < numbersToSort.length; i++) { if (!sorter.hasSpaceForAnotherRecord()) { sorter.expandPointerArray(consumer.allocateArray(sorter.getMemoryUsage() / 8 * 2)); } numbersToSort[i] = random.nextInt(PackedRecordPointer.MAXIMUM_PARTITION_ID + 1); sorter.insertRecord(0, numbersToSort[i]); } Arrays.sort(numbersToSort); int[] sorterResult = new int[numbersToSort.length]; ShuffleInMemorySorter.ShuffleSorterIterator iter = sorter.getSortedIterator(); int j = 0; while (iter.hasNext()) { iter.loadNext(); sorterResult[j] = iter.packedRecordPointer.getPartitionId(); j += 1; } Assert.assertArrayEquals(numbersToSort, sorterResult); } }
@Test public void heap() throws IOException { final SparkConf conf = new SparkConf().set("spark.memory.offHeap.enabled", "false"); final TaskMemoryManager memoryManager = new TaskMemoryManager(new TestMemoryManager(conf), 0); final MemoryConsumer c = new TestMemoryConsumer(memoryManager, MemoryMode.ON_HEAP); final MemoryBlock page0 = memoryManager.allocatePage(128, c); final MemoryBlock page1 = memoryManager.allocatePage(128, c); final long addressInPage1 = memoryManager.encodePageNumberAndOffset(page1, page1.getBaseOffset() + 42); PackedRecordPointer packedPointer = new PackedRecordPointer(); packedPointer.set(PackedRecordPointer.packPointer(addressInPage1, 360)); assertEquals(360, packedPointer.getPartitionId()); final long recordPointer = packedPointer.getRecordPointer(); assertEquals(1, TaskMemoryManager.decodePageNumber(recordPointer)); assertEquals(page1.getBaseOffset() + 42, memoryManager.getOffsetInPage(recordPointer)); assertEquals(addressInPage1, recordPointer); memoryManager.cleanUpAllAllocatedMemory(); }
@Test public void heap() throws IOException { final SparkConf conf = new SparkConf().set("spark.memory.offHeap.enabled", "false"); final TaskMemoryManager memoryManager = new TaskMemoryManager(new TestMemoryManager(conf), 0); final MemoryConsumer c = new TestMemoryConsumer(memoryManager, MemoryMode.ON_HEAP); final MemoryBlock page0 = memoryManager.allocatePage(128, c); final MemoryBlock page1 = memoryManager.allocatePage(128, c); final long addressInPage1 = memoryManager.encodePageNumberAndOffset(page1, page1.getBaseOffset() + 42); PackedRecordPointer packedPointer = new PackedRecordPointer(); packedPointer.set(PackedRecordPointer.packPointer(addressInPage1, 360)); assertEquals(360, packedPointer.getPartitionId()); final long recordPointer = packedPointer.getRecordPointer(); assertEquals(1, TaskMemoryManager.decodePageNumber(recordPointer)); assertEquals(page1.getBaseOffset() + 42, memoryManager.getOffsetInPage(recordPointer)); assertEquals(addressInPage1, recordPointer); memoryManager.cleanUpAllAllocatedMemory(); }
@Test public void heap() throws IOException { final SparkConf conf = new SparkConf().set("spark.memory.offHeap.enabled", "false"); final TaskMemoryManager memoryManager = new TaskMemoryManager(new TestMemoryManager(conf), 0); final MemoryConsumer c = new TestMemoryConsumer(memoryManager, MemoryMode.ON_HEAP); final MemoryBlock page0 = memoryManager.allocatePage(128, c); final MemoryBlock page1 = memoryManager.allocatePage(128, c); final long addressInPage1 = memoryManager.encodePageNumberAndOffset(page1, page1.getBaseOffset() + 42); PackedRecordPointer packedPointer = new PackedRecordPointer(); packedPointer.set(PackedRecordPointer.packPointer(addressInPage1, 360)); assertEquals(360, packedPointer.getPartitionId()); final long recordPointer = packedPointer.getRecordPointer(); assertEquals(1, TaskMemoryManager.decodePageNumber(recordPointer)); assertEquals(page1.getBaseOffset() + 42, memoryManager.getOffsetInPage(recordPointer)); assertEquals(addressInPage1, recordPointer); memoryManager.cleanUpAllAllocatedMemory(); }
@Test public void offHeap() throws IOException { final SparkConf conf = new SparkConf() .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "10000"); final TaskMemoryManager memoryManager = new TaskMemoryManager(new TestMemoryManager(conf), 0); final MemoryConsumer c = new TestMemoryConsumer(memoryManager, MemoryMode.OFF_HEAP); final MemoryBlock page0 = memoryManager.allocatePage(128, c); final MemoryBlock page1 = memoryManager.allocatePage(128, c); final long addressInPage1 = memoryManager.encodePageNumberAndOffset(page1, page1.getBaseOffset() + 42); PackedRecordPointer packedPointer = new PackedRecordPointer(); packedPointer.set(PackedRecordPointer.packPointer(addressInPage1, 360)); assertEquals(360, packedPointer.getPartitionId()); final long recordPointer = packedPointer.getRecordPointer(); assertEquals(1, TaskMemoryManager.decodePageNumber(recordPointer)); assertEquals(page1.getBaseOffset() + 42, memoryManager.getOffsetInPage(recordPointer)); assertEquals(addressInPage1, recordPointer); memoryManager.cleanUpAllAllocatedMemory(); }
@Test public void offHeap() throws IOException { final SparkConf conf = new SparkConf() .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "10000"); final TaskMemoryManager memoryManager = new TaskMemoryManager(new TestMemoryManager(conf), 0); final MemoryConsumer c = new TestMemoryConsumer(memoryManager, MemoryMode.OFF_HEAP); final MemoryBlock page0 = memoryManager.allocatePage(128, c); final MemoryBlock page1 = memoryManager.allocatePage(128, c); final long addressInPage1 = memoryManager.encodePageNumberAndOffset(page1, page1.getBaseOffset() + 42); PackedRecordPointer packedPointer = new PackedRecordPointer(); packedPointer.set(PackedRecordPointer.packPointer(addressInPage1, 360)); assertEquals(360, packedPointer.getPartitionId()); final long recordPointer = packedPointer.getRecordPointer(); assertEquals(1, TaskMemoryManager.decodePageNumber(recordPointer)); assertEquals(page1.getBaseOffset() + 42, memoryManager.getOffsetInPage(recordPointer)); assertEquals(addressInPage1, recordPointer); memoryManager.cleanUpAllAllocatedMemory(); }
@Test public void offHeap() throws IOException { final SparkConf conf = new SparkConf() .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "10000"); final TaskMemoryManager memoryManager = new TaskMemoryManager(new TestMemoryManager(conf), 0); final MemoryConsumer c = new TestMemoryConsumer(memoryManager, MemoryMode.OFF_HEAP); final MemoryBlock page0 = memoryManager.allocatePage(128, c); final MemoryBlock page1 = memoryManager.allocatePage(128, c); final long addressInPage1 = memoryManager.encodePageNumberAndOffset(page1, page1.getBaseOffset() + 42); PackedRecordPointer packedPointer = new PackedRecordPointer(); packedPointer.set(PackedRecordPointer.packPointer(addressInPage1, 360)); assertEquals(360, packedPointer.getPartitionId()); final long recordPointer = packedPointer.getRecordPointer(); assertEquals(1, TaskMemoryManager.decodePageNumber(recordPointer)); assertEquals(page1.getBaseOffset() + 42, memoryManager.getOffsetInPage(recordPointer)); assertEquals(addressInPage1, recordPointer); memoryManager.cleanUpAllAllocatedMemory(); }
Assert.assertTrue(iter.hasNext()); iter.loadNext(); final int partitionId = iter.packedRecordPointer.getPartitionId(); Assert.assertTrue(partitionId >= 0 && partitionId <= 3); Assert.assertTrue("Partition id " + partitionId + " should be >= prev id " + prevPartitionId,
Assert.assertTrue(iter.hasNext()); iter.loadNext(); final int partitionId = iter.packedRecordPointer.getPartitionId(); Assert.assertTrue(partitionId >= 0 && partitionId <= 3); Assert.assertTrue("Partition id " + partitionId + " should be >= prev id " + prevPartitionId,