public static Block createSlicesBlock(Slice... values) { requireNonNull(values, "varargs 'values' is null"); return createSlicesBlock(Arrays.asList(values)); }
private static Block createSliceArrayBlock(List<String> keys) { // last position is reserved for null Slice[] sliceArray = new Slice[keys.size() + 1]; for (int i = 0; i < keys.size(); i++) { sliceArray[i] = utf8Slice(keys.get(i)); } return createSlicesBlock(sliceArray); } }
private static Block createSliceArrayBlock(List<String> keys) { // last position is reserved for null Slice[] sliceArray = new Slice[keys.size() + 1]; for (int i = 0; i < keys.size(); i++) { sliceArray[i] = utf8Slice(keys.get(i)); } return createSlicesBlock(sliceArray); } }
private static DictionaryBlock createDictionaryBlock(Slice[] expectedValues, int positionCount) { int dictionarySize = expectedValues.length; int[] ids = new int[positionCount]; for (int i = 0; i < positionCount; i++) { ids[i] = i % dictionarySize; } return new DictionaryBlock(createSlicesBlock(expectedValues), ids); }
private static Block createSliceArrayBlock(List<String> values) { // last position is reserved for null Slice[] sliceArray = new Slice[values.size() + 1]; for (int i = 0; i < values.size(); i++) { sliceArray[i] = utf8Slice(values.get(i)); } return createSlicesBlock(sliceArray); }
private static Block createSliceArrayBlock(List<String> keys) { // last position is reserved for null Slice[] sliceArray = new Slice[keys.size() + 1]; for (int i = 0; i < keys.size(); i++) { sliceArray[i] = utf8Slice(keys.get(i)); } return createSlicesBlock(sliceArray); } }
private static DictionaryBlock createDictionaryBlock(Slice[] expectedValues, int positionCount) { int dictionarySize = expectedValues.length; int[] ids = new int[positionCount]; for (int i = 0; i < positionCount; i++) { ids[i] = i % dictionarySize; } return new DictionaryBlock(createSlicesBlock(expectedValues), ids); }
private static DictionaryBlock createDictionaryBlockWithUnreferencedKeys(Slice[] expectedValues, int positionCount) { // adds references to 0 and all odd indexes int dictionarySize = expectedValues.length; int[] ids = new int[positionCount]; for (int i = 0; i < positionCount; i++) { int index = i % dictionarySize; if (index % 2 == 0 && index != 0) { index--; } ids[i] = index; } return new DictionaryBlock(createSlicesBlock(expectedValues), ids); }
Page inputPage = new Page(createSlicesBlock(slices)); int batchSize = MAX_BATCH_SIZE; for (Optional<Page> actualPage : outputPages) { Page expectedPage = new Page(createSlicesBlock(Arrays.copyOfRange(slices, 0, batchSize))); assertPageEquals(ImmutableList.of(VARCHAR), actualPage.orElse(null), expectedPage); if (actualPage.orElseThrow(() -> new AssertionError("page is not present")).getSizeInBytes() > MAX_PAGE_SIZE_IN_BYTES) { inputPage = new Page(createSlicesBlock(slices)); int offset = 0; for (Optional<Page> actualPage : outputPages) { Page expectedPage = new Page(createSlicesBlock(Arrays.copyOfRange(slices, 0, Math.min(inputPage.getPositionCount() - offset, batchSize)))); assertPageEquals(ImmutableList.of(VARCHAR), actualPage.orElse(null), expectedPage); offset += actualPage.orElseThrow(() -> new AssertionError("page is not present")).getPositionCount();
Page inputPage = new Page(createSlicesBlock(slices)); Arrays.fill(blocks, createSlicesBlock(Arrays.copyOfRange(slices, 0, rows))); Page expectedPage = new Page(blocks); assertPageEquals(Collections.nCopies(columns, VARCHAR), actualPage, expectedPage);
@Test public void testDecreasingBatchSize() { int rows = 1024; // We set the expensive expression threshold to 0, so the expression is always considered expensive and the batch size gets halved until it becomes 1 TestingTicker testingTicker = new TestingTicker(); ExpressionProfiler profiler = new ExpressionProfiler(testingTicker, new Duration(0, MILLISECONDS)); PageProcessor pageProcessor = new PageProcessor( Optional.empty(), ImmutableList.of(new InputPageProjection(0, BIGINT)), OptionalInt.of(512), profiler); Slice[] slices = new Slice[rows]; Arrays.fill(slices, Slices.allocate(rows)); Page inputPage = new Page(createSlicesBlock(slices)); Iterator<Optional<Page>> output = processAndAssertRetainedPageSize(pageProcessor, inputPage); long previousPositionCount = 1; long totalPositionCount = 0; while (totalPositionCount < rows) { Optional<Page> page = output.next(); assertTrue(page.isPresent()); long positionCount = page.get().getPositionCount(); totalPositionCount += positionCount; // the batch size doesn't get smaller than 1 if (positionCount > 1 && previousPositionCount != 1) { assertEquals(positionCount, previousPositionCount / 2); } previousPositionCount = positionCount; } }
@Test public void testIncreasingBatchSize() { int rows = 1024; // We deliberately do not set the ticker, so that the expression is always cheap and the batch size gets doubled until other limits are hit TestingTicker testingTicker = new TestingTicker(); ExpressionProfiler profiler = new ExpressionProfiler(testingTicker, SPLIT_RUN_QUANTA); PageProcessor pageProcessor = new PageProcessor( Optional.empty(), ImmutableList.of(new InputPageProjection(0, BIGINT)), OptionalInt.of(1), profiler); Slice[] slices = new Slice[rows]; Arrays.fill(slices, Slices.allocate(rows)); Page inputPage = new Page(createSlicesBlock(slices)); Iterator<Optional<Page>> output = processAndAssertRetainedPageSize(pageProcessor, inputPage); long previousPositionCount = 1; long totalPositionCount = 0; while (totalPositionCount < rows) { Optional<Page> page = output.next(); assertTrue(page.isPresent()); long positionCount = page.get().getPositionCount(); totalPositionCount += positionCount; // skip the first read && skip the last read, which can be a partial page if (positionCount > 1 && totalPositionCount != rows) { assertEquals(positionCount, previousPositionCount * 2); } previousPositionCount = positionCount; } }
protected void assertAggregatedGeometries(String testDescription, String expectedWkt, String... wkts) { List<Slice> geometrySlices = Arrays.stream(wkts) .map(text -> text == null ? null : OGCGeometry.fromText(text)) .map(input -> input == null ? null : GeometrySerde.serialize(input)) .collect(Collectors.toList()); // Add a custom equality assertion because the resulting geometry may have // its constituent points in a different order BiFunction<Object, Object, Boolean> equalityFunction = (left, right) -> { if (left == null && right == null) { return true; } if (left == null || right == null) { return false; } OGCGeometry leftGeometry = OGCGeometry.fromText(left.toString()); OGCGeometry rightGeometry = OGCGeometry.fromText(right.toString()); // Check for equality by getting the difference return leftGeometry.difference(rightGeometry).isEmpty() && rightGeometry.difference(leftGeometry).isEmpty(); }; // Test in forward and reverse order to verify that ordering doesn't affect the output assertAggregation(function, equalityFunction, testDescription, new Page(BlockAssertions.createSlicesBlock(geometrySlices)), expectedWkt); Collections.reverse(geometrySlices); assertAggregation(function, equalityFunction, testDescription, new Page(BlockAssertions.createSlicesBlock(geometrySlices)), expectedWkt); }
Page inputPage = new Page(createSlicesBlock(slices)); while (output.hasNext()) { Page actualPage = output.next().orElse(null); Block sliceBlock = createSlicesBlock(Arrays.copyOfRange(slices, 0, batchSize)); Page expectedPage = new Page(sliceBlock, sliceBlock); assertPageEquals(ImmutableList.of(VARCHAR, VARCHAR), actualPage, expectedPage);
Block dictionaryBlock = new DictionaryBlock(createSlicesBlock(expectedValues), new int[] {0, 1, 2, 3, 4, 5}); assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] {expectedValues[0], expectedValues[1], expectedValues[2], expectedValues[3], expectedValues[4], expectedValues[5]}); DictionaryId dictionaryId = ((DictionaryBlock) dictionaryBlock).getDictionarySourceId();
@Test public void testCompactGetPositions() DictionaryBlock block = new DictionaryBlock(createSlicesBlock(createExpectedValues(10)), new int[] {0, 1, 2, 3, 4, 5}).compact();
public static Block createSlicesBlock(Slice... values) { requireNonNull(values, "varargs 'values' is null"); return createSlicesBlock(Arrays.asList(values)); }
protected void assertAggregatedGeometries(String testDescription, String expectedWkt, String... wkts) { List<Slice> geometrySlices = Arrays.stream(wkts) .map(text -> text == null ? null : OGCGeometry.fromText(text)) .map(input -> input == null ? null : GeometrySerde.serialize(input)) .collect(Collectors.toList()); // Add a custom equality assertion because the resulting geometry may have // its constituent points in a different order BiFunction<Object, Object, Boolean> equalityFunction = (left, right) -> { if (left == null && right == null) { return true; } if (left == null || right == null) { return false; } OGCGeometry leftGeometry = OGCGeometry.fromText(left.toString()); OGCGeometry rightGeometry = OGCGeometry.fromText(right.toString()); // Check for equality by getting the difference return leftGeometry.difference(rightGeometry).isEmpty() && rightGeometry.difference(leftGeometry).isEmpty(); }; // Test in forward and reverse order to verify that ordering doesn't affect the output assertAggregation(function, equalityFunction, testDescription, new Page(BlockAssertions.createSlicesBlock(geometrySlices)), expectedWkt); Collections.reverse(geometrySlices); assertAggregation(function, equalityFunction, testDescription, new Page(BlockAssertions.createSlicesBlock(geometrySlices)), expectedWkt); }