public RunLengthEncodedBlock(Block value, int positionCount) { requireNonNull(value, "value is null"); if (value.getPositionCount() != 1) { throw new IllegalArgumentException(format("Expected value to contain a single position but has %s positions", value.getPositionCount())); } if (value instanceof RunLengthEncodedBlock) { this.value = ((RunLengthEncodedBlock) value).getValue(); } else { this.value = value; } if (positionCount < 0) { throw new IllegalArgumentException("positionCount is negative"); } this.positionCount = positionCount; }
public static Block create(Type type, Object value, int positionCount) { Block block = Utils.nativeValueToBlock(type, value); if (block instanceof RunLengthEncodedBlock) { block = ((RunLengthEncodedBlock) block).getValue(); } return new RunLengthEncodedBlock(block, positionCount); }
public DictionaryAwarePageProjectionWork(@Nullable ConnectorSession session, DriverYieldSignal yieldSignal, Page page, SelectedPositions selectedPositions) { this.session = session; this.yieldSignal = requireNonNull(yieldSignal, "yieldSignal is null"); Block block = requireNonNull(page, "page is null").getBlock(0).getLoadedBlock(); this.block = block; this.selectedPositions = requireNonNull(selectedPositions, "selectedPositions is null"); Optional<Block> dictionary = Optional.empty(); if (block instanceof RunLengthEncodedBlock) { dictionary = Optional.of(((RunLengthEncodedBlock) block).getValue()); } else if (block instanceof DictionaryBlock) { dictionary = Optional.of(((DictionaryBlock) block).getDictionary()); } // Try use dictionary processing first; if it fails, fall back to the generic case dictionaryProcessingProjectionWork = createDictionaryBlockProjection(dictionary); fallbackProcessingProjectionWork = null; }
public DictionaryAwarePageProjectionWork(@Nullable ConnectorSession session, DriverYieldSignal yieldSignal, Page page, SelectedPositions selectedPositions) { this.session = session; this.yieldSignal = requireNonNull(yieldSignal, "yieldSignal is null"); Block block = requireNonNull(page, "page is null").getBlock(0).getLoadedBlock(); this.block = block; this.selectedPositions = requireNonNull(selectedPositions, "selectedPositions is null"); Optional<Block> dictionary = Optional.empty(); if (block instanceof RunLengthEncodedBlock) { dictionary = Optional.of(((RunLengthEncodedBlock) block).getValue()); } else if (block instanceof DictionaryBlock) { dictionary = Optional.of(((DictionaryBlock) block).getDictionary()); } // Try use dictionary processing first; if it fails, fall back to the generic case dictionaryProcessingProjectionWork = createDictionaryBlockProjection(dictionary); fallbackProcessingProjectionWork = null; }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) block; // write the run length sliceOutput.writeInt(rleBlock.getPositionCount()); // write the value blockEncodingSerde.writeBlock(sliceOutput, rleBlock.getValue()); }
@Override public SelectedPositions filter(ConnectorSession session, Page page) { Block block = page.getBlock(0).getLoadedBlock(); if (block instanceof RunLengthEncodedBlock) { Block value = ((RunLengthEncodedBlock) block).getValue(); Optional<boolean[]> selectedPosition = processDictionary(session, value); // single value block is always considered effective, but the processing could have thrown // in that case we fallback and process again so the correct error message sent if (selectedPosition.isPresent()) { return SelectedPositions.positionsRange(0, selectedPosition.get()[0] ? page.getPositionCount() : 0); } } if (block instanceof DictionaryBlock) { DictionaryBlock dictionaryBlock = (DictionaryBlock) block; // Attempt to process the dictionary. If dictionary is processing has not been considered effective, an empty response will be returned Optional<boolean[]> selectedDictionaryPositions = processDictionary(session, dictionaryBlock.getDictionary()); // record the usage count regardless of dictionary processing choice, so we have stats for next time lastDictionaryUsageCount += page.getPositionCount(); // if dictionary was processed, produce a dictionary block; otherwise do normal processing if (selectedDictionaryPositions.isPresent()) { return selectDictionaryPositions(dictionaryBlock, selectedDictionaryPositions.get()); } } return filter.filter(session, new Page(block)); }
@Override public SelectedPositions filter(ConnectorSession session, Page page) { Block block = page.getBlock(0).getLoadedBlock(); if (block instanceof RunLengthEncodedBlock) { Block value = ((RunLengthEncodedBlock) block).getValue(); Optional<boolean[]> selectedPosition = processDictionary(session, value); // single value block is always considered effective, but the processing could have thrown // in that case we fallback and process again so the correct error message sent if (selectedPosition.isPresent()) { return SelectedPositions.positionsRange(0, selectedPosition.get()[0] ? page.getPositionCount() : 0); } } if (block instanceof DictionaryBlock) { DictionaryBlock dictionaryBlock = (DictionaryBlock) block; // Attempt to process the dictionary. If dictionary is processing has not been considered effective, an empty response will be returned Optional<boolean[]> selectedDictionaryPositions = processDictionary(session, dictionaryBlock.getDictionary()); // record the usage count regardless of dictionary processing choice, so we have stats for next time lastDictionaryUsageCount += page.getPositionCount(); // if dictionary was processed, produce a dictionary block; otherwise do normal processing if (selectedDictionaryPositions.isPresent()) { return selectDictionaryPositions(dictionaryBlock, selectedDictionaryPositions.get()); } } return filter.filter(session, new Page(block)); }
private static ColumnarRow toColumnarRow(RunLengthEncodedBlock rleBlock) { Block rleValue = rleBlock.getValue(); ColumnarRow columnarRow = toColumnarRow(rleValue); Block[] fields = new Block[columnarRow.getFieldCount()]; for (int i = 0; i < columnarRow.getFieldCount(); i++) { Block nullSuppressedField = columnarRow.getField(i); if (rleValue.isNull(0)) { // the rle value is a null row so, all null-suppressed fields should empty if (nullSuppressedField.getPositionCount() != 0) { throw new IllegalArgumentException("Invalid row block"); } fields[i] = nullSuppressedField; } else { fields[i] = new RunLengthEncodedBlock(nullSuppressedField, rleBlock.getPositionCount()); } } return new ColumnarRow(rleBlock, fields); }
@Test public void testSanityFilterOnRLE() { Signature lessThan = internalOperator(LESS_THAN, BOOLEAN, ImmutableList.of(BIGINT, BIGINT)); CallExpression filter = new CallExpression(lessThan, BOOLEAN, ImmutableList.of(field(0, BIGINT), constant(10L, BIGINT))); PageProcessor processor = compiler.compilePageProcessor(Optional.of(filter), ImmutableList.of(field(0, BIGINT)), MAX_BATCH_SIZE).get(); Page page = new Page(createRLEBlock(5L, 100)); Page outputPage = getOnlyElement( processor.process( null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)) .orElseThrow(() -> new AssertionError("page is not present")); assertEquals(outputPage.getPositionCount(), 100); assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock); RunLengthEncodedBlock rle = (RunLengthEncodedBlock) outputPage.getBlock(0); assertEquals(BIGINT.getLong(rle.getValue(), 0), 5L); }
@Test public void testSanityFilterOnRLE() { Signature lessThan = internalOperator(LESS_THAN, BOOLEAN, ImmutableList.of(BIGINT, BIGINT)); CallExpression filter = new CallExpression(lessThan, BOOLEAN, ImmutableList.of(field(0, BIGINT), constant(10L, BIGINT))); PageProcessor processor = compiler.compilePageProcessor(Optional.of(filter), ImmutableList.of(field(0, BIGINT)), MAX_BATCH_SIZE).get(); Page page = new Page(createRLEBlock(5L, 100)); Page outputPage = getOnlyElement( processor.process( null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)) .orElseThrow(() -> new AssertionError("page is not present")); assertEquals(outputPage.getPositionCount(), 100); assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock); RunLengthEncodedBlock rle = (RunLengthEncodedBlock) outputPage.getBlock(0); assertEquals(BIGINT.getLong(rle.getValue(), 0), 5L); }
@Test public void testSanityRLE() { PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(field(0, BIGINT), field(1, VARCHAR)), MAX_BATCH_SIZE).get(); Slice varcharValue = Slices.utf8Slice("hello"); Page page = new Page(RunLengthEncodedBlock.create(BIGINT, 123L, 100), RunLengthEncodedBlock.create(VARCHAR, varcharValue, 100)); Page outputPage = getOnlyElement( processor.process( null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)) .orElseThrow(() -> new AssertionError("page is not present")); assertEquals(outputPage.getPositionCount(), 100); assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock); assertTrue(outputPage.getBlock(1) instanceof RunLengthEncodedBlock); RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) outputPage.getBlock(0); assertEquals(BIGINT.getLong(rleBlock.getValue(), 0), 123L); RunLengthEncodedBlock rleBlock1 = (RunLengthEncodedBlock) outputPage.getBlock(1); assertEquals(VARCHAR.getSlice(rleBlock1.getValue(), 0), varcharValue); }
@Test public void testSanityRLE() { PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(field(0, BIGINT), field(1, VARCHAR)), MAX_BATCH_SIZE).get(); Slice varcharValue = Slices.utf8Slice("hello"); Page page = new Page(RunLengthEncodedBlock.create(BIGINT, 123L, 100), RunLengthEncodedBlock.create(VARCHAR, varcharValue, 100)); Page outputPage = getOnlyElement( processor.process( null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)) .orElseThrow(() -> new AssertionError("page is not present")); assertEquals(outputPage.getPositionCount(), 100); assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock); assertTrue(outputPage.getBlock(1) instanceof RunLengthEncodedBlock); RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) outputPage.getBlock(0); assertEquals(BIGINT.getLong(rleBlock.getValue(), 0), 123L); RunLengthEncodedBlock rleBlock1 = (RunLengthEncodedBlock) outputPage.getBlock(1); assertEquals(VARCHAR.getSlice(rleBlock1.getValue(), 0), varcharValue); }
private static ColumnarArray toColumnarArray(RunLengthEncodedBlock rleBlock) { ColumnarArray columnarArray = toColumnarArray(rleBlock.getValue()); // build new offsets block int[] offsets = new int[rleBlock.getPositionCount() + 1]; int valueLength = columnarArray.getLength(0); for (int i = 0; i < offsets.length; i++) { offsets[i] = i * valueLength; } // create indexes for a dictionary block of the elements int[] dictionaryIds = new int[rleBlock.getPositionCount() * valueLength]; int nextDictionaryIndex = 0; for (int position = 0; position < rleBlock.getPositionCount(); position++) { for (int entryIndex = 0; entryIndex < valueLength; entryIndex++) { dictionaryIds[nextDictionaryIndex] = entryIndex; nextDictionaryIndex++; } } return new ColumnarArray( rleBlock, 0, offsets, new DictionaryBlock(dictionaryIds.length, columnarArray.getElementsBlock(), dictionaryIds)); }
private static ColumnarMap toColumnarMap(RunLengthEncodedBlock rleBlock) { ColumnarMap columnarMap = toColumnarMap(rleBlock.getValue()); // build new offsets block int[] offsets = new int[rleBlock.getPositionCount() + 1]; int entryCount = columnarMap.getEntryCount(0); for (int i = 0; i < offsets.length; i++) { offsets[i] = i * entryCount; } // create indexes for a dictionary block of the elements int[] dictionaryIds = new int[rleBlock.getPositionCount() * entryCount]; int nextDictionaryIndex = 0; for (int position = 0; position < rleBlock.getPositionCount(); position++) { for (int entryIndex = 0; entryIndex < entryCount; entryIndex++) { dictionaryIds[nextDictionaryIndex] = entryIndex; nextDictionaryIndex++; } } return new ColumnarMap( rleBlock, 0, offsets, new DictionaryBlock(dictionaryIds.length, columnarMap.getKeysBlock(), dictionaryIds), new DictionaryBlock(dictionaryIds.length, columnarMap.getValuesBlock(), dictionaryIds)); }