private static Block buildSingleValueBlock(Type type, Object value) { Block block = nativeValueToBlock(type, value); return new RunLengthEncodedBlock(block, MAX_BATCH_SIZE); }
private static RunLengthEncodedBlock createNullRLEBlock(int positionCount) { return (RunLengthEncodedBlock) RunLengthEncodedBlock.create(BOOLEAN, null, positionCount); }
public static Block create(Type type, Object value, int positionCount) { Block block = Utils.nativeValueToBlock(type, value); if (block instanceof RunLengthEncodedBlock) { block = ((RunLengthEncodedBlock) block).getValue(); } return new RunLengthEncodedBlock(block, positionCount); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) block; // write the run length sliceOutput.writeInt(rleBlock.getPositionCount()); // write the value blockEncodingSerde.writeBlock(sliceOutput, rleBlock.getValue()); }
private static ColumnarRow toColumnarRow(RunLengthEncodedBlock rleBlock) { Block rleValue = rleBlock.getValue(); ColumnarRow columnarRow = toColumnarRow(rleValue); Block[] fields = new Block[columnarRow.getFieldCount()]; for (int i = 0; i < columnarRow.getFieldCount(); i++) { Block nullSuppressedField = columnarRow.getField(i); if (rleValue.isNull(0)) { // the rle value is a null row so, all null-suppressed fields should empty if (nullSuppressedField.getPositionCount() != 0) { throw new IllegalArgumentException("Invalid row block"); } fields[i] = nullSuppressedField; } else { fields[i] = new RunLengthEncodedBlock(nullSuppressedField, rleBlock.getPositionCount()); } } return new ColumnarRow(rleBlock, fields); }
@Test public void testSanityRLE() { PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(field(0, BIGINT), field(1, VARCHAR)), MAX_BATCH_SIZE).get(); Slice varcharValue = Slices.utf8Slice("hello"); Page page = new Page(RunLengthEncodedBlock.create(BIGINT, 123L, 100), RunLengthEncodedBlock.create(VARCHAR, varcharValue, 100)); Page outputPage = getOnlyElement( processor.process( null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)) .orElseThrow(() -> new AssertionError("page is not present")); assertEquals(outputPage.getPositionCount(), 100); assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock); assertTrue(outputPage.getBlock(1) instanceof RunLengthEncodedBlock); RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) outputPage.getBlock(0); assertEquals(BIGINT.getLong(rleBlock.getValue(), 0), 123L); RunLengthEncodedBlock rleBlock1 = (RunLengthEncodedBlock) outputPage.getBlock(1); assertEquals(VARCHAR.getSlice(rleBlock1.getValue(), 0), varcharValue); }
public RunLengthEncodedBlock(Block value, int positionCount) { requireNonNull(value, "value is null"); if (value.getPositionCount() != 1) { throw new IllegalArgumentException(format("Expected value to contain a single position but has %s positions", value.getPositionCount())); } if (value instanceof RunLengthEncodedBlock) { this.value = ((RunLengthEncodedBlock) value).getValue(); } else { this.value = value; } if (positionCount < 0) { throw new IllegalArgumentException("positionCount is negative"); } this.positionCount = positionCount; }
@Override public Block getSingleValueBlock(int position) { checkReadablePosition(position); return value; }
@Test public void testSanityRLE() { PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(field(0, BIGINT), field(1, VARCHAR)), MAX_BATCH_SIZE).get(); Slice varcharValue = Slices.utf8Slice("hello"); Page page = new Page(RunLengthEncodedBlock.create(BIGINT, 123L, 100), RunLengthEncodedBlock.create(VARCHAR, varcharValue, 100)); Page outputPage = getOnlyElement( processor.process( null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)) .orElseThrow(() -> new AssertionError("page is not present")); assertEquals(outputPage.getPositionCount(), 100); assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock); assertTrue(outputPage.getBlock(1) instanceof RunLengthEncodedBlock); RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) outputPage.getBlock(0); assertEquals(BIGINT.getLong(rleBlock.getValue(), 0), 123L); RunLengthEncodedBlock rleBlock1 = (RunLengthEncodedBlock) outputPage.getBlock(1); assertEquals(VARCHAR.getSlice(rleBlock1.getValue(), 0), varcharValue); }
public DictionaryAwarePageProjectionWork(@Nullable ConnectorSession session, DriverYieldSignal yieldSignal, Page page, SelectedPositions selectedPositions) { this.session = session; this.yieldSignal = requireNonNull(yieldSignal, "yieldSignal is null"); Block block = requireNonNull(page, "page is null").getBlock(0).getLoadedBlock(); this.block = block; this.selectedPositions = requireNonNull(selectedPositions, "selectedPositions is null"); Optional<Block> dictionary = Optional.empty(); if (block instanceof RunLengthEncodedBlock) { dictionary = Optional.of(((RunLengthEncodedBlock) block).getValue()); } else if (block instanceof DictionaryBlock) { dictionary = Optional.of(((DictionaryBlock) block).getDictionary()); } // Try use dictionary processing first; if it fails, fall back to the generic case dictionaryProcessingProjectionWork = createDictionaryBlockProjection(dictionary); fallbackProcessingProjectionWork = null; }
private static ColumnarArray toColumnarArray(RunLengthEncodedBlock rleBlock) { ColumnarArray columnarArray = toColumnarArray(rleBlock.getValue()); // build new offsets block int[] offsets = new int[rleBlock.getPositionCount() + 1]; int valueLength = columnarArray.getLength(0); for (int i = 0; i < offsets.length; i++) { offsets[i] = i * valueLength; } // create indexes for a dictionary block of the elements int[] dictionaryIds = new int[rleBlock.getPositionCount() * valueLength]; int nextDictionaryIndex = 0; for (int position = 0; position < rleBlock.getPositionCount(); position++) { for (int entryIndex = 0; entryIndex < valueLength; entryIndex++) { dictionaryIds[nextDictionaryIndex] = entryIndex; nextDictionaryIndex++; } } return new ColumnarArray( rleBlock, 0, offsets, new DictionaryBlock(dictionaryIds.length, columnarArray.getElementsBlock(), dictionaryIds)); }
@Override public void writePositionTo(int position, BlockBuilder blockBuilder) { checkReadablePosition(position); value.writePositionTo(0, blockBuilder); }
@Override public Block getLoadedBlock() { Block loadedValueBlock = value.getLoadedBlock(); if (loadedValueBlock == value) { return this; } return new RunLengthEncodedBlock(loadedValueBlock, positionCount); }
private static RunLengthEncodedBlock createNullRLEBlock(int positionCount) { return (RunLengthEncodedBlock) RunLengthEncodedBlock.create(BOOLEAN, null, positionCount); }
public DictionaryAwarePageProjectionWork(@Nullable ConnectorSession session, DriverYieldSignal yieldSignal, Page page, SelectedPositions selectedPositions) { this.session = session; this.yieldSignal = requireNonNull(yieldSignal, "yieldSignal is null"); Block block = requireNonNull(page, "page is null").getBlock(0).getLoadedBlock(); this.block = block; this.selectedPositions = requireNonNull(selectedPositions, "selectedPositions is null"); Optional<Block> dictionary = Optional.empty(); if (block instanceof RunLengthEncodedBlock) { dictionary = Optional.of(((RunLengthEncodedBlock) block).getValue()); } else if (block instanceof DictionaryBlock) { dictionary = Optional.of(((DictionaryBlock) block).getDictionary()); } // Try use dictionary processing first; if it fails, fall back to the generic case dictionaryProcessingProjectionWork = createDictionaryBlockProjection(dictionary); fallbackProcessingProjectionWork = null; }
private static ColumnarMap toColumnarMap(RunLengthEncodedBlock rleBlock) { ColumnarMap columnarMap = toColumnarMap(rleBlock.getValue()); // build new offsets block int[] offsets = new int[rleBlock.getPositionCount() + 1]; int entryCount = columnarMap.getEntryCount(0); for (int i = 0; i < offsets.length; i++) { offsets[i] = i * entryCount; } // create indexes for a dictionary block of the elements int[] dictionaryIds = new int[rleBlock.getPositionCount() * entryCount]; int nextDictionaryIndex = 0; for (int position = 0; position < rleBlock.getPositionCount(); position++) { for (int entryIndex = 0; entryIndex < entryCount; entryIndex++) { dictionaryIds[nextDictionaryIndex] = entryIndex; nextDictionaryIndex++; } } return new ColumnarMap( rleBlock, 0, offsets, new DictionaryBlock(dictionaryIds.length, columnarMap.getKeysBlock(), dictionaryIds), new DictionaryBlock(dictionaryIds.length, columnarMap.getValuesBlock(), dictionaryIds)); }
@Override public byte getByte(int position, int offset) { checkReadablePosition(position); return value.getByte(0, offset); }
@Override public Block build() { if (!hasNonNullValue) { return new RunLengthEncodedBlock(NULL_VALUE_BLOCK, positionCount); } return new ByteArrayBlock(0, positionCount, hasNullValue ? valueIsNull : null, values); }
private Page createStatisticsPage(Page aggregationOutput) { int positionCount = aggregationOutput.getPositionCount(); Block[] outputBlocks = new Block[types.size()]; for (int channel = 0; channel < types.size(); channel++) { if (channel < STATS_START_CHANNEL) { outputBlocks[channel] = RunLengthEncodedBlock.create(types.get(channel), null, positionCount); } else { outputBlocks[channel] = aggregationOutput.getBlock(channel - 2); } } return new Page(positionCount, outputBlocks); }
@Override public SelectedPositions filter(ConnectorSession session, Page page) { Block block = page.getBlock(0).getLoadedBlock(); if (block instanceof RunLengthEncodedBlock) { Block value = ((RunLengthEncodedBlock) block).getValue(); Optional<boolean[]> selectedPosition = processDictionary(session, value); // single value block is always considered effective, but the processing could have thrown // in that case we fallback and process again so the correct error message sent if (selectedPosition.isPresent()) { return SelectedPositions.positionsRange(0, selectedPosition.get()[0] ? page.getPositionCount() : 0); } } if (block instanceof DictionaryBlock) { DictionaryBlock dictionaryBlock = (DictionaryBlock) block; // Attempt to process the dictionary. If dictionary is processing has not been considered effective, an empty response will be returned Optional<boolean[]> selectedDictionaryPositions = processDictionary(session, dictionaryBlock.getDictionary()); // record the usage count regardless of dictionary processing choice, so we have stats for next time lastDictionaryUsageCount += page.getPositionCount(); // if dictionary was processed, produce a dictionary block; otherwise do normal processing if (selectedDictionaryPositions.isPresent()) { return selectDictionaryPositions(dictionaryBlock, selectedDictionaryPositions.get()); } } return filter.filter(session, new Page(block)); }