@Test public void testCopyPositionsNoCompaction() { Slice[] expectedValues = createExpectedValues(1); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); int[] positionsToCopy = new int[] {0, 2, 4, 5}; DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy, 0, positionsToCopy.length); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.length); assertBlock(copiedBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, expectedValues); }
@Test public void testCompact() { Slice[] expectedValues = createExpectedValues(5); DictionaryBlock dictionaryBlock = createDictionaryBlockWithUnreferencedKeys(expectedValues, 10); assertEquals(dictionaryBlock.isCompact(), false); DictionaryBlock compactBlock = dictionaryBlock.compact(); assertNotEquals(dictionaryBlock.getDictionarySourceId(), compactBlock.getDictionarySourceId()); assertEquals(compactBlock.getDictionary().getPositionCount(), (expectedValues.length / 2) + 1); assertBlock(compactBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, new Slice[] {expectedValues[0], expectedValues[1], expectedValues[3]}); assertDictionaryIds(compactBlock, 0, 1, 1, 2, 2, 0, 1, 1, 2, 2); assertEquals(compactBlock.isCompact(), true); DictionaryBlock reCompactedBlock = compactBlock.compact(); assertEquals(reCompactedBlock.getDictionarySourceId(), compactBlock.getDictionarySourceId()); }
@Test public void testLogicalSizeInBytes() { // The 10 Slices in the array will be of lengths 0 to 9. Slice[] expectedValues = createExpectedValues(10); // The dictionary within the dictionary block is expected to be a VariableWidthBlock of size 95 bytes. // 45 bytes for the expectedValues Slices (sum of seq(0,9)) and 50 bytes for the position and isNull array (total 10 positions). DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); assertEquals(dictionaryBlock.getDictionary().getLogicalSizeInBytes(), 95); // The 100 positions in the dictionary block index to 10 positions in the underlying dictionary (10 each). // Logical size calculation accounts for 4 bytes of offset and 1 byte of isNull. Therefore the expected unoptimized // size is 10 times the size of the underlying dictionary (VariableWidthBlock). assertEquals(dictionaryBlock.getLogicalSizeInBytes(), 95 * 10); // With alternating nulls, we have 21 positions, with the same size calculation as above. dictionaryBlock = createDictionaryBlock(alternatingNullValues(expectedValues), 210); assertEquals(dictionaryBlock.getDictionary().getPositionCount(), 21); assertEquals(dictionaryBlock.getDictionary().getLogicalSizeInBytes(), 150); // The null positions should be included in the logical size. assertEquals(dictionaryBlock.getLogicalSizeInBytes(), 150 * 10); }
@Test public void testCopyRegionCreatesCompactBlock() { Slice[] expectedValues = createExpectedValues(10); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); DictionaryBlock copyRegionDictionaryBlock = (DictionaryBlock) dictionaryBlock.copyRegion(1, 3); assertTrue(copyRegionDictionaryBlock.isCompact()); }
@Test public void testCopyPositionsWithCompactionsAndReorder() { Slice[] expectedValues = createExpectedValues(10); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); int[] positionsToCopy = new int[] {50, 55, 40, 45, 60}; DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy, 0, positionsToCopy.length); assertEquals(copiedBlock.getDictionary().getPositionCount(), 2); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.length); assertBlock(copiedBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, new Slice[] {expectedValues[0], expectedValues[5]}); assertDictionaryIds(copiedBlock, 0, 1, 0, 1, 0); }
@Test public void testCompact() throws Exception { Slice[] expectedValues = createExpectedValues(5); DictionaryBlock dictionaryBlock = createDictionaryBlockWithUnreferencedKeys(expectedValues, 10); assertEquals(dictionaryBlock.isCompact(), false); DictionaryBlock compactBlock = dictionaryBlock.compact(); assertNotEquals(dictionaryBlock.getDictionarySourceId(), compactBlock.getDictionarySourceId()); assertEquals(compactBlock.getDictionary().getPositionCount(), (expectedValues.length / 2) + 1); assertBlock(compactBlock.getDictionary(), new Slice[] { expectedValues[0], expectedValues[1], expectedValues[3] }); assertEquals(compactBlock.getIds(), wrappedIntArray(0, 1, 1, 2, 2, 0, 1, 1, 2, 2)); assertEquals(compactBlock.isCompact(), true); DictionaryBlock reCompactedBlock = compactBlock.compact(); assertEquals(reCompactedBlock.getDictionarySourceId(), compactBlock.getDictionarySourceId()); }
@Test public void testBasicGetPositions() Slice[] expectedValues = createExpectedValues(10); Block dictionaryBlock = new DictionaryBlock(createSlicesBlock(expectedValues), new int[] {0, 1, 2, 3, 4, 5}); assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] {expectedValues[0], expectedValues[1], expectedValues[2], expectedValues[3], expectedValues[4], expectedValues[5]}); DictionaryId dictionaryId = ((DictionaryBlock) dictionaryBlock).getDictionarySourceId(); assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] {expectedValues[1], expectedValues[2], expectedValues[4], expectedValues[5]}); assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId); assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] {expectedValues[1], expectedValues[2], expectedValues[5]}); assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId); assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] {expectedValues[1], expectedValues[2], expectedValues[5]}); assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId); assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] {expectedValues[1], expectedValues[5], expectedValues[5]}); assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId); assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] {expectedValues[5], expectedValues[5], expectedValues[5], expectedValues[5], expectedValues[5]}); assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId);
@Test public void testCompactGetPositions() DictionaryBlock block = new DictionaryBlock(createSlicesBlock(createExpectedValues(10)), new int[] {0, 1, 2, 3, 4, 5}).compact();
@Test public void testEstimatedDataSizeForStats() { int positionCount = 10; int dictionaryPositionCount = 100; Slice[] expectedValues = createExpectedValues(positionCount); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, dictionaryPositionCount); for (int position = 0; position < dictionaryPositionCount; position++) { assertEquals(dictionaryBlock.getEstimatedDataSizeForStats(position), expectedValues[position % positionCount].length()); } }
@Test public void testCopyPositionsSamePosition() { Slice[] expectedValues = createExpectedValues(10); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); int[] positionsToCopy = new int[] {52, 52, 52}; DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy, 0, positionsToCopy.length); assertEquals(copiedBlock.getDictionary().getPositionCount(), 1); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.length); assertBlock(copiedBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, new Slice[] {expectedValues[2]}); assertDictionaryIds(copiedBlock, 0, 0, 0); }
@Test public void testCopyPositionsWithCompaction() { Slice[] expectedValues = createExpectedValues(10); Slice firstExpectedValue = expectedValues[0]; DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); int[] positionsToCopy = new int[] {0, 10, 20, 30, 40}; DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy, 0, positionsToCopy.length); assertEquals(copiedBlock.getDictionary().getPositionCount(), 1); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.length); assertBlock(copiedBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, new Slice[] {firstExpectedValue}); assertBlock(copiedBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] {firstExpectedValue, firstExpectedValue, firstExpectedValue, firstExpectedValue, firstExpectedValue}); }
@Test public void testSizeInBytes() { Slice[] expectedValues = createExpectedValues(10); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); assertEquals(dictionaryBlock.getSizeInBytes(), dictionaryBlock.getDictionary().getSizeInBytes() + (100 * SIZE_OF_INT)); }
@Test public void testCopyPositionsNoCompaction() throws Exception { Slice[] expectedValues = createExpectedValues(1); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); List<Integer> positionsToCopy = Ints.asList(0, 2, 4, 5); DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.size()); assertBlock(copiedBlock.getDictionary(), expectedValues); }
@Test public void testCompactAllKeysReferenced() { Slice[] expectedValues = createExpectedValues(5); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 10); DictionaryBlock compactBlock = dictionaryBlock.compact(); // When there is nothing to compact, we return the same block assertEquals(compactBlock.getDictionary(), dictionaryBlock.getDictionary()); assertEquals(compactBlock.getPositionCount(), dictionaryBlock.getPositionCount()); for (int position = 0; position < compactBlock.getPositionCount(); position++) { assertEquals(compactBlock.getId(position), dictionaryBlock.getId(position)); } assertEquals(compactBlock.isCompact(), true); }
@Test public void testCopyPositionsWithCompaction() throws Exception { Slice[] expectedValues = createExpectedValues(10); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); List<Integer> positionsToCopy = Ints.asList(0, 10, 20, 30, 40); DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy); assertEquals(copiedBlock.getDictionary().getPositionCount(), 1); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.size()); assertBlock(copiedBlock.getDictionary(), Arrays.copyOfRange(expectedValues, 0, 1)); }
@Test public void testSizeInBytes() throws Exception { Slice[] expectedValues = createExpectedValues(10); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); int sizeInBytes = 0; for (Slice expectedValue : expectedValues) { sizeInBytes += expectedValue.length(); } assertEquals(dictionaryBlock.getSizeInBytes(), sizeInBytes + (100 * SIZE_OF_INT)); }
@Test public void testCopyPositionsSamePosition() throws Exception { Slice[] expectedValues = createExpectedValues(10); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); List<Integer> positionsToCopy = Ints.asList(52, 52, 52); DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy); assertEquals(copiedBlock.getDictionary().getPositionCount(), 1); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.size()); assertBlock(copiedBlock.getDictionary(), new Slice[] { expectedValues[2] }); assertEquals(copiedBlock.getIds(), wrappedIntArray(0, 0, 0)); }
@Test public void testCopyRegionCreatesCompactBlock() throws Exception { Slice[] expectedValues = createExpectedValues(10); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); DictionaryBlock copyRegionDictionaryBlock = (DictionaryBlock) dictionaryBlock.copyRegion(1, 3); assertTrue(copyRegionDictionaryBlock.isCompact()); }
@Test public void testCopyPositionsWithCompactionsAndReorder() throws Exception { Slice[] expectedValues = createExpectedValues(10); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); List<Integer> positionsToCopy = Ints.asList(50, 55, 40, 45, 60); DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy); assertEquals(copiedBlock.getDictionary().getPositionCount(), 2); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.size()); assertBlock(copiedBlock.getDictionary(), new Slice[] { expectedValues[0], expectedValues[5] }); assertEquals(copiedBlock.getIds(), wrappedIntArray(0, 1, 0, 1, 0)); }
@Test public void testCompactAllKeysReferenced() throws Exception { Slice[] expectedValues = createExpectedValues(5); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 10); DictionaryBlock compactBlock = dictionaryBlock.compact(); // When there is nothing to compact, we return the same block assertEquals(compactBlock.getDictionary(), dictionaryBlock.getDictionary()); assertEquals(compactBlock.getIds(), dictionaryBlock.getIds()); assertEquals(compactBlock.isCompact(), true); }