@Override public Block apply(Block block) { ColumnarRow rowBlock = toColumnarRow(block); Block[] fields = new Block[coercers.length]; int[] ids = new int[rowBlock.getField(0).getPositionCount()]; for (int i = 0; i < coercers.length; i++) { if (coercers[i] != null) { fields[i] = coercers[i].apply(rowBlock.getField(i)); } else if (i < rowBlock.getFieldCount()) { fields[i] = rowBlock.getField(i); } else { fields[i] = new DictionaryBlock(nullBlocks[i], ids); } } boolean[] valueIsNull = new boolean[rowBlock.getPositionCount()]; for (int i = 0; i < rowBlock.getPositionCount(); i++) { valueIsNull[i] = rowBlock.isNull(i); } return RowBlock.fromFieldBlocks(valueIsNull.length, Optional.of(valueIsNull), fields); } }
private static ColumnarRow toColumnarRow(RunLengthEncodedBlock rleBlock) { Block rleValue = rleBlock.getValue(); ColumnarRow columnarRow = toColumnarRow(rleValue); Block[] fields = new Block[columnarRow.getFieldCount()]; for (int i = 0; i < columnarRow.getFieldCount(); i++) { Block nullSuppressedField = columnarRow.getField(i); if (rleValue.isNull(0)) { // the rle value is a null row so, all null-suppressed fields should empty if (nullSuppressedField.getPositionCount() != 0) { throw new IllegalArgumentException("Invalid row block"); } fields[i] = nullSuppressedField; } else { fields[i] = new RunLengthEncodedBlock(nullSuppressedField, rleBlock.getPositionCount()); } } return new ColumnarRow(rleBlock, fields); }
@Override public void deserialize(Block block, int index, MultimapAggregationState state) { state.reset(); ColumnarRow columnarRow = toColumnarRow(arrayType.getObject(block, index)); Block keys = columnarRow.getField(KEY_CHANNEL); Block values = columnarRow.getField(VALUE_CHANNEL); for (int i = 0; i < columnarRow.getPositionCount(); i++) { state.add(keys, values, i); } } }
@Override public void appendNext(PageBuilder pageBuilder, int outputChannelOffset) { checkState(columnarRow != null, "columnarRow is null"); for (int i = 0; i < fieldTypes.size(); i++) { BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(outputChannelOffset + i); if (columnarRow.isNull(position)) { blockBuilder.appendNull(); } else { fieldTypes.get(i).appendTo(columnarRow.getField(i), nonNullPosition, blockBuilder); } } if (!columnarRow.isNull(position)) { nonNullPosition++; } position++; }
private void writeColumnarRow(ColumnarRow columnarRow) { // record nulls for (int position = 0; position < columnarRow.getPositionCount(); position++) { boolean present = !columnarRow.isNull(position); presentStream.writeBoolean(present); if (present) { nonNullValueCount++; } } // write field values for (int i = 0; i < structFields.size(); i++) { ColumnWriter columnWriter = structFields.get(i); Block fieldBlock = columnarRow.getField(i); if (fieldBlock.getPositionCount() > 0) { columnWriter.writeBlock(fieldBlock); } } }
statisticsBuilder = new CountStatisticsBuilder(); fieldExtractor = block -> { ColumnarRow columnarRow = ColumnarRow.toColumnarRow(block); ImmutableList.Builder<Block> fields = ImmutableList.builder(); for (int index = 0; index < columnarRow.getFieldCount(); index++) { fields.add(columnarRow.getField(index));
@Override public void deserialize(Block block, int index, T state) { checkArgument(block instanceof AbstractRowBlock); ColumnarRow columnarRow = toColumnarRow(block); state.setFirstNull(BOOLEAN.getBoolean(columnarRow.getField(0), index)); state.setSecondNull(BOOLEAN.getBoolean(columnarRow.getField(1), index)); if (!state.isFirstNull()) { readFirstField(columnarRow.getField(2), index, state); } if (!state.isSecondNull()) { state.setSecondPosition(index); state.setSecondBlock(columnarRow.getField(3)); } } }
@Override public void setBlock(Block block) { this.position = 0; this.nonNullPosition = 0; if (block == null) { this.columnarRow = null; this.positionCount = 0; } else { this.columnarRow = ColumnarRow.toColumnarRow(block); this.positionCount = block.getPositionCount(); } } }
public static ColumnarRow toColumnarRow(Block block) { requireNonNull(block, "block is null"); if (block instanceof DictionaryBlock) { return toColumnarRow((DictionaryBlock) block); } if (block instanceof RunLengthEncodedBlock) { return toColumnarRow((RunLengthEncodedBlock) block); } if (!(block instanceof AbstractRowBlock)) { throw new IllegalArgumentException("Invalid row block: " + block.getClass().getName()); } AbstractRowBlock rowBlock = (AbstractRowBlock) block; // get fields for visible region int firstRowPosition = rowBlock.getFieldBlockOffset(0); int totalRowCount = rowBlock.getFieldBlockOffset(block.getPositionCount()) - firstRowPosition; Block[] fieldBlocks = new Block[rowBlock.numFields]; for (int i = 0; i < fieldBlocks.length; i++) { fieldBlocks[i] = rowBlock.getRawFieldBlocks()[i].getRegion(firstRowPosition, totalRowCount); } return new ColumnarRow(block, fieldBlocks); }
private void writeColumnarRow(ColumnarRow columnarRow) { // record nulls for (int position = 0; position < columnarRow.getPositionCount(); position++) { boolean present = !columnarRow.isNull(position); presentStream.writeBoolean(present); if (present) { nonNullValueCount++; } } // write field values for (int i = 0; i < structFields.size(); i++) { ColumnWriter columnWriter = structFields.get(i); Block fieldBlock = columnarRow.getField(i); if (fieldBlock.getPositionCount() > 0) { columnWriter.writeBlock(fieldBlock); } } }
statisticsBuilder = new CountStatisticsBuilder(); fieldExtractor = block -> { ColumnarRow columnarRow = ColumnarRow.toColumnarRow(block); ImmutableList.Builder<Block> fields = ImmutableList.builder(); for (int index = 0; index < columnarRow.getFieldCount(); index++) { fields.add(columnarRow.getField(index));
@Override public void writeBlock(Block block) { checkState(!closed); checkArgument(block.getPositionCount() > 0, "Block is empty"); ColumnarRow columnarRow = toColumnarRow(block); writeColumnarRow(columnarRow); }
public static ColumnarRow toColumnarRow(Block block) { requireNonNull(block, "block is null"); if (block instanceof DictionaryBlock) { return toColumnarRow((DictionaryBlock) block); } if (block instanceof RunLengthEncodedBlock) { return toColumnarRow((RunLengthEncodedBlock) block); } if (!(block instanceof AbstractRowBlock)) { throw new IllegalArgumentException("Invalid row block: " + block.getClass().getName()); } AbstractRowBlock rowBlock = (AbstractRowBlock) block; // get fields for visible region int firstRowPosition = rowBlock.getFieldBlockOffset(0); int totalRowCount = rowBlock.getFieldBlockOffset(block.getPositionCount()) - firstRowPosition; Block[] fieldBlocks = new Block[rowBlock.numFields]; for (int i = 0; i < fieldBlocks.length; i++) { fieldBlocks[i] = rowBlock.getRawFieldBlocks()[i].getRegion(firstRowPosition, totalRowCount); } return new ColumnarRow(block, fieldBlocks); }
private static <T> void assertColumnarRow(Block block, T[] expectedValues) { ColumnarRow columnarRow = toColumnarRow(block); assertEquals(columnarRow.getPositionCount(), expectedValues.length); for (int fieldId = 0; fieldId < FIELD_COUNT; fieldId++) { Block fieldBlock = columnarRow.getField(fieldId); int elementsPosition = 0; for (int position = 0; position < expectedValues.length; position++) { T expectedRow = expectedValues[position]; assertEquals(columnarRow.isNull(position), expectedRow == null); if (expectedRow == null) { continue; } Object expectedElement = Array.get(expectedRow, fieldId); assertBlockPosition(fieldBlock, elementsPosition, expectedElement); elementsPosition++; } } }
private static ColumnarRow toColumnarRow(DictionaryBlock dictionaryBlock) { // build a mapping from the old dictionary to a new dictionary with nulls removed Block dictionary = dictionaryBlock.getDictionary(); int[] newDictionaryIndex = new int[dictionary.getPositionCount()]; int nextNewDictionaryIndex = 0; for (int position = 0; position < dictionary.getPositionCount(); position++) { if (!dictionary.isNull(position)) { newDictionaryIndex[position] = nextNewDictionaryIndex; nextNewDictionaryIndex++; } } // reindex the dictionary int[] dictionaryIds = new int[dictionaryBlock.getPositionCount()]; int nonNullPositionCount = 0; for (int position = 0; position < dictionaryBlock.getPositionCount(); position++) { if (!dictionaryBlock.isNull(position)) { int oldDictionaryId = dictionaryBlock.getId(position); dictionaryIds[nonNullPositionCount] = newDictionaryIndex[oldDictionaryId]; nonNullPositionCount++; } } ColumnarRow columnarRow = toColumnarRow(dictionaryBlock.getDictionary()); Block[] fields = new Block[columnarRow.getFieldCount()]; for (int i = 0; i < columnarRow.getFieldCount(); i++) { fields[i] = new DictionaryBlock(nonNullPositionCount, columnarRow.getField(i), dictionaryIds); } return new ColumnarRow(dictionaryBlock, fields); }
@Override public void writeBlock(Block block) { checkState(!closed); checkArgument(block.getPositionCount() > 0, "Block is empty"); ColumnarRow columnarRow = toColumnarRow(block); writeColumnarRow(columnarRow); }
private static ColumnarRow toColumnarRow(RunLengthEncodedBlock rleBlock) { Block rleValue = rleBlock.getValue(); ColumnarRow columnarRow = toColumnarRow(rleValue); Block[] fields = new Block[columnarRow.getFieldCount()]; for (int i = 0; i < columnarRow.getFieldCount(); i++) { Block nullSuppressedField = columnarRow.getField(i); if (rleValue.isNull(0)) { // the rle value is a null row so, all null-suppressed fields should empty if (nullSuppressedField.getPositionCount() != 0) { throw new IllegalArgumentException("Invalid row block"); } fields[i] = nullSuppressedField; } else { fields[i] = new RunLengthEncodedBlock(nullSuppressedField, rleBlock.getPositionCount()); } } return new ColumnarRow(rleBlock, fields); }
private static ColumnarRow toColumnarRow(DictionaryBlock dictionaryBlock) { // build a mapping from the old dictionary to a new dictionary with nulls removed Block dictionary = dictionaryBlock.getDictionary(); int[] newDictionaryIndex = new int[dictionary.getPositionCount()]; int nextNewDictionaryIndex = 0; for (int position = 0; position < dictionary.getPositionCount(); position++) { if (!dictionary.isNull(position)) { newDictionaryIndex[position] = nextNewDictionaryIndex; nextNewDictionaryIndex++; } } // reindex the dictionary int[] dictionaryIds = new int[dictionaryBlock.getPositionCount()]; int nonNullPositionCount = 0; for (int position = 0; position < dictionaryBlock.getPositionCount(); position++) { if (!dictionaryBlock.isNull(position)) { int oldDictionaryId = dictionaryBlock.getId(position); dictionaryIds[nonNullPositionCount] = newDictionaryIndex[oldDictionaryId]; nonNullPositionCount++; } } ColumnarRow columnarRow = toColumnarRow(dictionaryBlock.getDictionary()); Block[] fields = new Block[columnarRow.getFieldCount()]; for (int i = 0; i < columnarRow.getFieldCount(); i++) { fields[i] = new DictionaryBlock(nonNullPositionCount, columnarRow.getField(i), dictionaryIds); } return new ColumnarRow(dictionaryBlock, fields); }