/** * Create a row block directly without per element validations. */ static RowBlock createRowBlockInternal(int startOffset, int positionCount, @Nullable boolean[] rowIsNull, int[] fieldBlockOffsets, Block[] fieldBlocks) { validateConstructorArguments(startOffset, positionCount, rowIsNull, fieldBlockOffsets, fieldBlocks); return new RowBlock(startOffset, positionCount, rowIsNull, fieldBlockOffsets, fieldBlocks); }
@Override public long getSizeInBytes() { if (sizeInBytes < 0) { calculateSize(); } return sizeInBytes; }
@Override public Block build() { if (currentEntryOpened) { throw new IllegalStateException("Current entry must be closed before the block can be built"); } Block[] fieldBlocks = new Block[numFields]; for (int i = 0; i < numFields; i++) { fieldBlocks[i] = fieldBlockBuilders[i].build(); } return createRowBlockInternal(0, positionCount, rowIsNull, fieldBlockOffsets, fieldBlocks); }
Block rowBlock = RowBlock.fromFieldBlocks(nextBatchSize, Optional.ofNullable(nullVector), blocks);
@Override public String toString() { return format("RowBlock{numFields=%d, positionCount=%d}", numFields, getPositionCount()); }
private ColumnChunk readStruct(GroupField field) throws IOException { List<TypeSignatureParameter> fields = field.getType().getTypeSignature().getParameters(); Block[] blocks = new Block[fields.size()]; ColumnChunk columnChunk = null; List<Optional<Field>> parameters = field.getChildren(); for (int i = 0; i < fields.size(); i++) { Optional<Field> parameter = parameters.get(i); if (parameter.isPresent()) { columnChunk = readColumnChunk(parameter.get()); blocks[i] = columnChunk.getBlock(); } } for (int i = 0; i < fields.size(); i++) { if (blocks[i] == null) { blocks[i] = RunLengthEncodedBlock.create(field.getType(), null, columnChunk.getBlock().getPositionCount()); } } BooleanList structIsNull = StructColumnReader.calculateStructOffsets(field, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); boolean[] structIsNullVector = structIsNull.toBooleanArray(); Block rowBlock = RowBlock.fromFieldBlocks(structIsNullVector.length, Optional.of(structIsNullVector), blocks); return new ColumnChunk(rowBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); }
Block rowBlock = RowBlock.fromFieldBlocks(nextBatchSize, Optional.ofNullable(nullVector), blocks);
/** * Create a row block directly from columnar nulls and field blocks. */ public static Block fromFieldBlocks(int positionCount, Optional<boolean[]> rowIsNull, Block[] fieldBlocks) { int[] fieldBlockOffsets = new int[positionCount + 1]; for (int position = 0; position < positionCount; position++) { fieldBlockOffsets[position + 1] = fieldBlockOffsets[position] + (rowIsNull.isPresent() && rowIsNull.get()[position] ? 0 : 1); } validateConstructorArguments(0, positionCount, rowIsNull.orElse(null), fieldBlockOffsets, fieldBlocks); return new RowBlock(0, positionCount, rowIsNull.orElse(null), fieldBlockOffsets, fieldBlocks); }
@Override public Block getLoadedBlock() { boolean allLoaded = true; Block[] loadedFieldBlocks = new Block[fieldBlocks.length]; for (int i = 0; i < fieldBlocks.length; i++) { loadedFieldBlocks[i] = fieldBlocks[i].getLoadedBlock(); if (loadedFieldBlocks[i] != fieldBlocks[i]) { allLoaded = false; } } if (allLoaded) { return this; } return createRowBlockInternal( startOffset, positionCount, rowIsNull, fieldBlockOffsets, loadedFieldBlocks); } }
private ColumnChunk readStruct(GroupField field) throws IOException { List<TypeSignatureParameter> fields = field.getType().getTypeSignature().getParameters(); Block[] blocks = new Block[fields.size()]; ColumnChunk columnChunk = null; List<Optional<Field>> parameters = field.getChildren(); for (int i = 0; i < fields.size(); i++) { Optional<Field> parameter = parameters.get(i); if (parameter.isPresent()) { columnChunk = readColumnChunk(parameter.get()); blocks[i] = columnChunk.getBlock(); } } for (int i = 0; i < fields.size(); i++) { if (blocks[i] == null) { blocks[i] = RunLengthEncodedBlock.create(field.getType(), null, columnChunk.getBlock().getPositionCount()); } } BooleanList structIsNull = StructColumnReader.calculateStructOffsets(field, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); boolean[] structIsNullVector = structIsNull.toBooleanArray(); Block rowBlock = RowBlock.fromFieldBlocks(structIsNullVector.length, Optional.of(structIsNullVector), blocks); return new ColumnChunk(rowBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); }
@Override public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { int numFields = sliceInput.readInt(); Block[] fieldBlocks = new Block[numFields]; for (int i = 0; i < numFields; i++) { fieldBlocks[i] = blockEncodingSerde.readBlock(sliceInput); } int positionCount = sliceInput.readInt(); int[] fieldBlockOffsets = new int[positionCount + 1]; sliceInput.readBytes(wrappedIntArray(fieldBlockOffsets)); boolean[] rowIsNull = EncoderUtil.decodeNullBits(sliceInput, positionCount).orElseGet(() -> new boolean[positionCount]); return createRowBlockInternal(0, positionCount, rowIsNull, fieldBlockOffsets, fieldBlocks); } }
@Test public void testCompactBlock() { Block emptyBlock = new ByteArrayBlock(0, Optional.empty(), new byte[0]); Block compactFieldBlock1 = new ByteArrayBlock(5, Optional.empty(), createExpectedValue(5).getBytes()); Block compactFieldBlock2 = new ByteArrayBlock(5, Optional.empty(), createExpectedValue(5).getBytes()); Block incompactFiledBlock1 = new ByteArrayBlock(5, Optional.empty(), createExpectedValue(6).getBytes()); Block incompactFiledBlock2 = new ByteArrayBlock(5, Optional.empty(), createExpectedValue(6).getBytes()); boolean[] rowIsNull = {false, true, false, false, false, false}; assertCompact(fromFieldBlocks(0, Optional.empty(), new Block[] {emptyBlock, emptyBlock})); assertCompact(fromFieldBlocks(rowIsNull.length, Optional.of(rowIsNull), new Block[] {compactFieldBlock1, compactFieldBlock2})); // TODO: add test case for a sliced RowBlock // underlying field blocks are not compact testIncompactBlock(fromFieldBlocks(rowIsNull.length, Optional.of(rowIsNull), new Block[] {incompactFiledBlock1, incompactFiledBlock2})); testIncompactBlock(fromFieldBlocks(rowIsNull.length, Optional.of(rowIsNull), new Block[] {incompactFiledBlock1, incompactFiledBlock2})); }
@Override public Block getRegion(int position, int length) { int positionCount = getPositionCount(); checkValidRegion(positionCount, position, length); return createRowBlockInternal(position + getOffsetBase(), length, getRowIsNull(), getFieldBlockOffsets(), getRawFieldBlocks()); }
@Test public void testCompactBlock() { Block emptyBlock = new ByteArrayBlock(0, Optional.empty(), new byte[0]); Block compactFieldBlock1 = new ByteArrayBlock(5, Optional.empty(), createExpectedValue(5).getBytes()); Block compactFieldBlock2 = new ByteArrayBlock(5, Optional.empty(), createExpectedValue(5).getBytes()); Block incompactFiledBlock1 = new ByteArrayBlock(5, Optional.empty(), createExpectedValue(6).getBytes()); Block incompactFiledBlock2 = new ByteArrayBlock(5, Optional.empty(), createExpectedValue(6).getBytes()); boolean[] rowIsNull = {false, true, false, false, false, false}; assertCompact(fromFieldBlocks(0, Optional.empty(), new Block[] {emptyBlock, emptyBlock})); assertCompact(fromFieldBlocks(rowIsNull.length, Optional.of(rowIsNull), new Block[] {compactFieldBlock1, compactFieldBlock2})); // TODO: add test case for a sliced RowBlock // underlying field blocks are not compact testIncompactBlock(fromFieldBlocks(rowIsNull.length, Optional.of(rowIsNull), new Block[] {incompactFiledBlock1, incompactFiledBlock2})); testIncompactBlock(fromFieldBlocks(rowIsNull.length, Optional.of(rowIsNull), new Block[] {incompactFiledBlock1, incompactFiledBlock2})); }
@Override public Block getSingleValueBlock(int position) { checkReadablePosition(position); int startFieldBlockOffset = getFieldBlockOffset(position); int endFieldBlockOffset = getFieldBlockOffset(position + 1); int fieldBlockLength = endFieldBlockOffset - startFieldBlockOffset; Block[] newBlocks = new Block[numFields]; for (int i = 0; i < numFields; i++) { newBlocks[i] = getRawFieldBlocks()[i].copyRegion(startFieldBlockOffset, fieldBlockLength); } boolean[] newRowIsNull = new boolean[] {isNull(position)}; int[] newOffsets = new int[] {0, fieldBlockLength}; return createRowBlockInternal(0, 1, newRowIsNull, newOffsets, newBlocks); }
@Override public Block apply(Block block) { ColumnarRow rowBlock = toColumnarRow(block); Block[] fields = new Block[coercers.length]; int[] ids = new int[rowBlock.getField(0).getPositionCount()]; for (int i = 0; i < coercers.length; i++) { if (coercers[i] != null) { fields[i] = coercers[i].apply(rowBlock.getField(i)); } else if (i < rowBlock.getFieldCount()) { fields[i] = rowBlock.getField(i); } else { fields[i] = new DictionaryBlock(nullBlocks[i], ids); } } boolean[] valueIsNull = new boolean[rowBlock.getPositionCount()]; for (int i = 0; i < rowBlock.getPositionCount(); i++) { valueIsNull[i] = rowBlock.isNull(i); } return RowBlock.fromFieldBlocks(valueIsNull.length, Optional.of(valueIsNull), fields); } }
@Override public Block copyPositions(int[] positions, int offset, int length) { checkArrayRange(positions, offset, length); int[] newOffsets = new int[length + 1]; boolean[] newRowIsNull = new boolean[length]; IntArrayList fieldBlockPositions = new IntArrayList(length); for (int i = 0; i < length; i++) { int position = positions[offset + i]; if (isNull(position)) { newRowIsNull[i] = true; newOffsets[i + 1] = newOffsets[i]; } else { newOffsets[i + 1] = newOffsets[i] + 1; fieldBlockPositions.add(getFieldBlockOffset(position)); } } Block[] newBlocks = new Block[numFields]; for (int i = 0; i < numFields; i++) { newBlocks[i] = getRawFieldBlocks()[i].copyPositions(fieldBlockPositions.elements(), 0, fieldBlockPositions.size()); } return createRowBlockInternal(0, length, newRowIsNull, newOffsets, newBlocks); }
@Setup public void setup() { Signature signature = new Signature("$operator$CAST", SCALAR, RowType.anonymous(fromFieldTypes).getTypeSignature(), RowType.anonymous(toFieldTypes).getTypeSignature()); List<RowExpression> projections = ImmutableList.of( new CallExpression(signature, RowType.anonymous(fromFieldTypes), ImmutableList.of(field(0, RowType.anonymous(toFieldTypes))))); MetadataManager metadata = MetadataManager.createTestMetadataManager(); pageProcessor = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)) .compilePageProcessor(Optional.empty(), projections) .get(); Block[] fieldBlocks = fromFieldTypes.stream() .map(type -> createBlock(POSITION_COUNT, type)) .toArray(Block[]::new); Block rowBlock = fromFieldBlocks(POSITION_COUNT, Optional.empty(), fieldBlocks); page = new Page(rowBlock); }
@Override public Block copyRegion(int position, int length) { int positionCount = getPositionCount(); checkValidRegion(positionCount, position, length); int startFieldBlockOffset = getFieldBlockOffset(position); int endFieldBlockOffset = getFieldBlockOffset(position + length); int fieldBlockLength = endFieldBlockOffset - startFieldBlockOffset; Block[] newBlocks = new Block[numFields]; for (int i = 0; i < numFields; i++) { newBlocks[i] = getRawFieldBlocks()[i].copyRegion(startFieldBlockOffset, fieldBlockLength); } int[] newOffsets = compactOffsets(getFieldBlockOffsets(), position + getOffsetBase(), length); boolean[] rowIsNull = getRowIsNull(); boolean[] newRowIsNull = rowIsNull == null ? null : compactArray(rowIsNull, position + getOffsetBase(), length); if (arraySame(newBlocks, getRawFieldBlocks()) && newOffsets == getFieldBlockOffsets() && newRowIsNull == rowIsNull) { return this; } return createRowBlockInternal(0, length, newRowIsNull, newOffsets, newBlocks); }
@Setup public void setup() { Signature signature = new Signature("$operator$CAST", SCALAR, RowType.anonymous(fromFieldTypes).getTypeSignature(), RowType.anonymous(toFieldTypes).getTypeSignature()); List<RowExpression> projections = ImmutableList.of( new CallExpression(signature, RowType.anonymous(fromFieldTypes), ImmutableList.of(field(0, RowType.anonymous(toFieldTypes))))); MetadataManager metadata = MetadataManager.createTestMetadataManager(); pageProcessor = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)) .compilePageProcessor(Optional.empty(), projections) .get(); Block[] fieldBlocks = fromFieldTypes.stream() .map(type -> createBlock(POSITION_COUNT, type)) .toArray(Block[]::new); Block rowBlock = fromFieldBlocks(POSITION_COUNT, Optional.empty(), fieldBlocks); page = new Page(rowBlock); }