Refine search
private Page getPartitionFunctionArguments(Page page) { Block[] blocks = new Block[partitionChannels.size()]; for (int i = 0; i < blocks.length; i++) { Optional<Block> partitionConstant = partitionConstants.get(i); if (partitionConstant.isPresent()) { blocks[i] = new RunLengthEncodedBlock(partitionConstant.get(), page.getPositionCount()); } else { blocks[i] = page.getBlock(partitionChannels.get(i)); } } return new Page(page.getPositionCount(), blocks); }
private Page createStatisticsPage(Page aggregationOutput) { int positionCount = aggregationOutput.getPositionCount(); Block[] outputBlocks = new Block[types.size()]; for (int channel = 0; channel < types.size(); channel++) { if (channel < STATS_START_CHANNEL) { outputBlocks[channel] = RunLengthEncodedBlock.create(types.get(channel), null, positionCount); } else { outputBlocks[channel] = aggregationOutput.getBlock(channel - 2); } } return new Page(positionCount, outputBlocks); }
public DictionaryAwarePageProjectionWork(@Nullable ConnectorSession session, DriverYieldSignal yieldSignal, Page page, SelectedPositions selectedPositions) { this.session = session; this.yieldSignal = requireNonNull(yieldSignal, "yieldSignal is null"); Block block = requireNonNull(page, "page is null").getBlock(0).getLoadedBlock(); this.block = block; this.selectedPositions = requireNonNull(selectedPositions, "selectedPositions is null"); Optional<Block> dictionary = Optional.empty(); if (block instanceof RunLengthEncodedBlock) { dictionary = Optional.of(((RunLengthEncodedBlock) block).getValue()); } else if (block instanceof DictionaryBlock) { dictionary = Optional.of(((DictionaryBlock) block).getDictionary()); } // Try use dictionary processing first; if it fails, fall back to the generic case dictionaryProcessingProjectionWork = createDictionaryBlockProjection(dictionary); fallbackProcessingProjectionWork = null; }
public static Block create(Type type, Object value, int positionCount) { Block block = Utils.nativeValueToBlock(type, value); if (block instanceof RunLengthEncodedBlock) { block = ((RunLengthEncodedBlock) block).getValue(); } return new RunLengthEncodedBlock(block, positionCount); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) block; // write the run length sliceOutput.writeInt(rleBlock.getPositionCount()); // write the value blockEncodingSerde.writeBlock(sliceOutput, rleBlock.getValue()); }
private static ColumnarRow toColumnarRow(RunLengthEncodedBlock rleBlock) { Block rleValue = rleBlock.getValue(); ColumnarRow columnarRow = toColumnarRow(rleValue); Block[] fields = new Block[columnarRow.getFieldCount()]; for (int i = 0; i < columnarRow.getFieldCount(); i++) { Block nullSuppressedField = columnarRow.getField(i); if (rleValue.isNull(0)) { // the rle value is a null row so, all null-suppressed fields should empty if (nullSuppressedField.getPositionCount() != 0) { throw new IllegalArgumentException("Invalid row block"); } fields[i] = nullSuppressedField; } else { fields[i] = new RunLengthEncodedBlock(nullSuppressedField, rleBlock.getPositionCount()); } } return new ColumnarRow(rleBlock, fields); }
Optional<Block> dictionaryOutput = Optional.empty(); if (dictionaryProcessingProjectionWork != null) { try { dictionaryOutput = Optional.of(dictionaryProcessingProjectionWork.getResult()); lastOutputDictionary = dictionaryOutput; lastOutputDictionary = Optional.empty(); dictionaryProcessingProjectionWork = null; result = new RunLengthEncodedBlock(dictionaryOutput.get(), selectedPositions.size()); return true; fallbackProcessingProjectionWork = projection.project(session, yieldSignal, new Page(block), selectedPositions); if (fallbackProcessingProjectionWork.process()) { result = fallbackProcessingProjectionWork.getResult();
private ColumnChunk readStruct(GroupField field) throws IOException { List<TypeSignatureParameter> fields = field.getType().getTypeSignature().getParameters(); Block[] blocks = new Block[fields.size()]; ColumnChunk columnChunk = null; List<Optional<Field>> parameters = field.getChildren(); for (int i = 0; i < fields.size(); i++) { Optional<Field> parameter = parameters.get(i); if (parameter.isPresent()) { columnChunk = readColumnChunk(parameter.get()); blocks[i] = columnChunk.getBlock(); } } for (int i = 0; i < fields.size(); i++) { if (blocks[i] == null) { blocks[i] = RunLengthEncodedBlock.create(field.getType(), null, columnChunk.getBlock().getPositionCount()); } } BooleanList structIsNull = StructColumnReader.calculateStructOffsets(field, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); boolean[] structIsNullVector = structIsNull.toBooleanArray(); Block rowBlock = RowBlock.fromFieldBlocks(structIsNullVector.length, Optional.of(structIsNullVector), blocks); return new ColumnChunk(rowBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); }
@Override public SelectedPositions filter(ConnectorSession session, Page page) { Block block = page.getBlock(0).getLoadedBlock(); if (block instanceof RunLengthEncodedBlock) { Block value = ((RunLengthEncodedBlock) block).getValue(); Optional<boolean[]> selectedPosition = processDictionary(session, value); // single value block is always considered effective, but the processing could have thrown // in that case we fallback and process again so the correct error message sent if (selectedPosition.isPresent()) { return SelectedPositions.positionsRange(0, selectedPosition.get()[0] ? page.getPositionCount() : 0); } } if (block instanceof DictionaryBlock) { DictionaryBlock dictionaryBlock = (DictionaryBlock) block; // Attempt to process the dictionary. If dictionary is processing has not been considered effective, an empty response will be returned Optional<boolean[]> selectedDictionaryPositions = processDictionary(session, dictionaryBlock.getDictionary()); // record the usage count regardless of dictionary processing choice, so we have stats for next time lastDictionaryUsageCount += page.getPositionCount(); // if dictionary was processed, produce a dictionary block; otherwise do normal processing if (selectedDictionaryPositions.isPresent()) { return selectDictionaryPositions(dictionaryBlock, selectedDictionaryPositions.get()); } } return filter.filter(session, new Page(block)); }
if (bucketAdapter.isPresent()) { IntArrayList rowsToKeep = bucketAdapter.get().computeEligibleRowIds(dataPage); Block[] adaptedBlocks = new Block[dataPage.getChannelCount()]; for (int i = 0; i < adaptedBlocks.length; i++) { Block block = dataPage.getBlock(i); if (block instanceof LazyBlock && !((LazyBlock) block).isLoaded()) { adaptedBlocks[i] = new LazyBlock(rowsToKeep.size(), new RowFilterLazyBlockLoader(dataPage.getBlock(i), rowsToKeep)); switch (columnMapping.getKind()) { case PREFILLED: blocks.add(RunLengthEncodedBlock.create(types[fieldId], prefilledValues[fieldId], batchSize)); break; case REGULAR:
@Test public void testSanityRLE() { PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(field(0, BIGINT), field(1, VARCHAR)), MAX_BATCH_SIZE).get(); Slice varcharValue = Slices.utf8Slice("hello"); Page page = new Page(RunLengthEncodedBlock.create(BIGINT, 123L, 100), RunLengthEncodedBlock.create(VARCHAR, varcharValue, 100)); Page outputPage = getOnlyElement( processor.process( null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)) .orElseThrow(() -> new AssertionError("page is not present")); assertEquals(outputPage.getPositionCount(), 100); assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock); assertTrue(outputPage.getBlock(1) instanceof RunLengthEncodedBlock); RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) outputPage.getBlock(0); assertEquals(BIGINT.getLong(rleBlock.getValue(), 0), 123L); RunLengthEncodedBlock rleBlock1 = (RunLengthEncodedBlock) outputPage.getBlock(1); assertEquals(VARCHAR.getSlice(rleBlock1.getValue(), 0), varcharValue); }
@Test public void testSanityFilterOnRLE() { Signature lessThan = internalOperator(LESS_THAN, BOOLEAN, ImmutableList.of(BIGINT, BIGINT)); CallExpression filter = new CallExpression(lessThan, BOOLEAN, ImmutableList.of(field(0, BIGINT), constant(10L, BIGINT))); PageProcessor processor = compiler.compilePageProcessor(Optional.of(filter), ImmutableList.of(field(0, BIGINT)), MAX_BATCH_SIZE).get(); Page page = new Page(createRLEBlock(5L, 100)); Page outputPage = getOnlyElement( processor.process( null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)) .orElseThrow(() -> new AssertionError("page is not present")); assertEquals(outputPage.getPositionCount(), 100); assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock); RunLengthEncodedBlock rle = (RunLengthEncodedBlock) outputPage.getBlock(0); assertEquals(BIGINT.getLong(rle.getValue(), 0), 5L); }
@Test(dataProvider = "forceYield") public void testRleBlock(boolean forceYield) { Block value = createLongSequenceBlock(42, 43); RunLengthEncodedBlock block = new RunLengthEncodedBlock(value, 100); testProject(block, RunLengthEncodedBlock.class, forceYield); }
@Test public void testAllPositionsNull() { // if there are no parameters skip this test List<Type> parameterTypes = getFunction().getParameterTypes(); if (parameterTypes.isEmpty()) { return; } Block[] blocks = new Block[parameterTypes.size()]; for (int i = 0; i < parameterTypes.size(); i++) { blocks[i] = RunLengthEncodedBlock.create(parameterTypes.get(0), null, 10); } testAggregation(getExpectedValueIncludingNulls(0, 0, 10), blocks); }
private static RunLengthEncodedBlock createNullRLEBlock(int positionCount) { Block value = BOOLEAN.createBlockBuilder(new BlockBuilderStatus(), 1) .appendNull() .build(); return new RunLengthEncodedBlock(value, positionCount); }
@Test public void testSanityRLE() throws Exception { PageProcessor processor = new ExpressionCompiler(createTestMetadataManager()) .compilePageProcessor(new ConstantExpression(TRUE, BOOLEAN), ImmutableList.of(new InputReferenceExpression(0, BIGINT), new InputReferenceExpression(1, VARCHAR))).get(); Slice varcharValue = Slices.utf8Slice("hello"); Page page = new Page(RunLengthEncodedBlock.create(BIGINT, 123L, 100), RunLengthEncodedBlock.create(VARCHAR, varcharValue, 100)); Page outputPage = processor.processColumnarDictionary(null, page, ImmutableList.of(BIGINT, VARCHAR)); assertEquals(outputPage.getPositionCount(), 100); assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock); assertTrue(outputPage.getBlock(1) instanceof RunLengthEncodedBlock); RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) outputPage.getBlock(0); assertEquals(BIGINT.getLong(rleBlock.getValue(), 0), 123L); RunLengthEncodedBlock rleBlock1 = (RunLengthEncodedBlock) outputPage.getBlock(1); assertEquals(VARCHAR.getSlice(rleBlock1.getValue(), 0), varcharValue); }
@Test private void testSplitPageNonDecreasingPageSize() { int positionCount = 100; int maxPageSizeInBytes = 1; List<Type> types = ImmutableList.of(VARCHAR); Slice expectedValue = wrappedBuffer("test".getBytes()); BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 1, expectedValue.length()); blockBuilder.writeBytes(expectedValue, 0, expectedValue.length()).closeEntry(); Block rleBlock = new RunLengthEncodedBlock(blockBuilder.build(), positionCount); Page initialPage = new Page(rleBlock); List<Page> pages = splitPage(initialPage, maxPageSizeInBytes); // the page should only be split in half as the recursion should terminate // after seeing that the size of the Page doesn't decrease assertEquals(pages.size(), 2); Page first = pages.get(0); Page second = pages.get(1); // the size of the pages will remain the same and should be greater than the maxPageSizeInBytes assertGreaterThan((int) first.getSizeInBytes(), maxPageSizeInBytes); assertGreaterThan((int) second.getSizeInBytes(), maxPageSizeInBytes); assertPositionCount(pages, positionCount); MaterializedResult actual = toMaterializedResult(TEST_SESSION, types, pages); MaterializedResult expected = toMaterializedResult(TEST_SESSION, types, ImmutableList.of(initialPage)); assertEquals(actual, expected); } }
@Test public void testRunLengthEncodedBlock() { BlockBuilder blockBuilder = new LongArrayBlockBuilder(null, 1); writeEntries(1, blockBuilder, BIGINT); checkRetainedSize(new RunLengthEncodedBlock(blockBuilder.build(), 1), false); }