/** * Create an array block directly without per element validations. */ static ArrayBlock createArrayBlockInternal(int arrayOffset, int positionCount, @Nullable boolean[] valueIsNull, int[] offsets, Block values) { validateConstructorArguments(arrayOffset, positionCount, valueIsNull, offsets, values); return new ArrayBlock(arrayOffset, positionCount, valueIsNull, offsets, values); }
@Override public long getSizeInBytes() { if (sizeInBytes < 0) { calculateSize(); } return sizeInBytes; }
@Override public ArrayBlock build() { if (currentEntryOpened) { throw new IllegalStateException("Current entry must be closed before the block can be built"); } return createArrayBlockInternal(0, positionCount, hasNullValue ? valueIsNull : null, offsets, values.build()); }
@Override public Block toBlock(Type desiredType) { checkArgument(desiredType.getTypeParameters().size() == 1 && BIGINT.equals(desiredType.getTypeParameters().get(0)), "type doesn't match: %s", desiredType); int numberOfRecords = numberOfRecords(); return ArrayBlock.fromElementBlock( numberOfRecords, Optional.of(nulls == null ? new boolean[numberOfRecords] : nulls), calculateOffsets(sizes, nulls, numberOfRecords), values != null ? values.toBlock(BIGINT) : new LongArrayBlock(0, Optional.empty(), new long[] {})); }
@Override public String toString() { StringBuilder sb = new StringBuilder("ArrayBlock{"); sb.append("positionCount=").append(getPositionCount()); sb.append('}'); return sb.toString(); }
private static Block createArrayBlock(int positionCount, Block elementsBlock) { int[] offsets = new int[positionCount + 1]; int arraySize = elementsBlock.getPositionCount() / positionCount; for (int i = 0; i < offsets.length; i++) { offsets[i] = arraySize * i; } return ArrayBlock.fromElementBlock(positionCount, Optional.empty(), offsets, elementsBlock); }
private static Block createArrayBlock(int positionCount, Block elementsBlock) { int[] offsets = new int[positionCount + 1]; int arraySize = elementsBlock.getPositionCount() / positionCount; for (int i = 0; i < offsets.length; i++) { offsets[i] = arraySize * i; } return ArrayBlock.fromElementBlock(positionCount, Optional.empty(), offsets, elementsBlock); }
/** * Create an array block directly from columnar nulls, values, and offsets into the values. * A null array must have no entries. */ public static Block fromElementBlock(int positionCount, Optional<boolean[]> valueIsNull, int[] arrayOffset, Block values) { validateConstructorArguments(0, positionCount, valueIsNull.orElse(null), arrayOffset, values); // for performance reasons per element checks are only performed on the public construction for (int i = 0; i < positionCount; i++) { int offset = arrayOffset[i]; int length = arrayOffset[i + 1] - offset; if (length < 0) { throw new IllegalArgumentException(format("Offset is not monotonically ascending. offsets[%s]=%s, offsets[%s]=%s", i, arrayOffset[i], i + 1, arrayOffset[i + 1])); } if (valueIsNull.isPresent() && valueIsNull.get()[i] && length != 0) { throw new IllegalArgumentException("A null array must have zero entries"); } } return new ArrayBlock(0, positionCount, valueIsNull.orElse(null), arrayOffset, values); }
@Override public Block getLoadedBlock() { Block loadedValuesBlock = values.getLoadedBlock(); if (loadedValuesBlock == values) { return this; } return createArrayBlockInternal( arrayOffset, positionCount, valueIsNull, offsets, loadedValuesBlock); } }
private ColumnChunk readArray(GroupField field) throws IOException { List<Type> parameters = field.getType().getTypeParameters(); checkArgument(parameters.size() == 1, "Arrays must have a single type parameter, found %d", parameters.size()); Field elementField = field.getChildren().get(0).get(); ColumnChunk columnChunk = readColumnChunk(elementField); IntList offsets = new IntArrayList(); BooleanList valueIsNull = new BooleanArrayList(); calculateCollectionOffsets(field, offsets, valueIsNull, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); Block arrayBlock = ArrayBlock.fromElementBlock(valueIsNull.size(), Optional.of(valueIsNull.toBooleanArray()), offsets.toIntArray(), columnChunk.getBlock()); return new ColumnChunk(arrayBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); }
@Override public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { Block values = blockEncodingSerde.readBlock(sliceInput); int positionCount = sliceInput.readInt(); int[] offsets = new int[positionCount + 1]; sliceInput.readBytes(Slices.wrappedIntArray(offsets)); boolean[] valueIsNull = decodeNullBits(sliceInput, positionCount).orElseGet(() -> new boolean[positionCount]); return createArrayBlockInternal(0, positionCount, valueIsNull, offsets, values); } }
private ColumnChunk readArray(GroupField field) throws IOException { List<Type> parameters = field.getType().getTypeParameters(); checkArgument(parameters.size() == 1, "Arrays must have a single type parameter, found %d", parameters.size()); Field elementField = field.getChildren().get(0).get(); ColumnChunk columnChunk = readColumnChunk(elementField); IntList offsets = new IntArrayList(); BooleanList valueIsNull = new BooleanArrayList(); calculateCollectionOffsets(field, offsets, valueIsNull, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); Block arrayBlock = ArrayBlock.fromElementBlock(valueIsNull.size(), Optional.of(valueIsNull.toBooleanArray()), offsets.toIntArray(), columnChunk.getBlock()); return new ColumnChunk(arrayBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); }
@Override public Block getRegion(int position, int length) { int positionCount = getPositionCount(); checkValidRegion(positionCount, position, length); return createArrayBlockInternal( position + getOffsetBase(), length, getValueIsNull(), getOffsets(), getRawElementBlock()); }
public void testCompactBlock() { Block emptyValueBlock = new ByteArrayBlock(0, Optional.empty(), new byte[0]); Block compactValueBlock = new ByteArrayBlock(16, Optional.empty(), createExpectedValue(16).getBytes()); Block inCompactValueBlock = new ByteArrayBlock(16, Optional.empty(), createExpectedValue(17).getBytes()); int[] offsets = {0, 1, 1, 2, 4, 8, 16}; boolean[] valueIsNull = {false, true, false, false, false, false}; testCompactBlock(fromElementBlock(0, Optional.empty(), new int[1], emptyValueBlock)); testCompactBlock(fromElementBlock(valueIsNull.length, Optional.of(valueIsNull), offsets, compactValueBlock)); testIncompactBlock(fromElementBlock(valueIsNull.length - 1, Optional.of(valueIsNull), offsets, compactValueBlock)); // underlying value block is not compact testIncompactBlock(fromElementBlock(valueIsNull.length, Optional.of(valueIsNull), offsets, inCompactValueBlock)); }
@Override public Block getSingleValueBlock(int position) { checkReadablePosition(position); int startValueOffset = getOffset(position); int valueLength = getOffset(position + 1) - startValueOffset; Block newValues = getRawElementBlock().copyRegion(startValueOffset, valueLength); return createArrayBlockInternal( 0, 1, new boolean[] {isNull(position)}, new int[] {0, valueLength}, newValues); }
public void testCompactBlock() { Block emptyValueBlock = new ByteArrayBlock(0, Optional.empty(), new byte[0]); Block compactValueBlock = new ByteArrayBlock(16, Optional.empty(), createExpectedValue(16).getBytes()); Block inCompactValueBlock = new ByteArrayBlock(16, Optional.empty(), createExpectedValue(17).getBytes()); int[] offsets = {0, 1, 1, 2, 4, 8, 16}; boolean[] valueIsNull = {false, true, false, false, false, false}; testCompactBlock(fromElementBlock(0, Optional.empty(), new int[1], emptyValueBlock)); testCompactBlock(fromElementBlock(valueIsNull.length, Optional.of(valueIsNull), offsets, compactValueBlock)); testIncompactBlock(fromElementBlock(valueIsNull.length - 1, Optional.of(valueIsNull), offsets, compactValueBlock)); // underlying value block is not compact testIncompactBlock(fromElementBlock(valueIsNull.length, Optional.of(valueIsNull), offsets, inCompactValueBlock)); }
@Override public Block copyPositions(int[] positions, int offset, int length) { checkArrayRange(positions, offset, length); int[] newOffsets = new int[length + 1]; boolean[] newValueIsNull = new boolean[length]; IntArrayList valuesPositions = new IntArrayList(); int newPosition = 0; for (int i = offset; i < offset + length; ++i) { int position = positions[i]; if (isNull(position)) { newValueIsNull[newPosition] = true; newOffsets[newPosition + 1] = newOffsets[newPosition]; } else { int valuesStartOffset = getOffset(position); int valuesEndOffset = getOffset(position + 1); int valuesLength = valuesEndOffset - valuesStartOffset; newOffsets[newPosition + 1] = newOffsets[newPosition] + valuesLength; for (int elementIndex = valuesStartOffset; elementIndex < valuesEndOffset; elementIndex++) { valuesPositions.add(elementIndex); } } newPosition++; } Block newValues = getRawElementBlock().copyPositions(valuesPositions.elements(), 0, valuesPositions.size()); return createArrayBlockInternal(0, length, newValueIsNull, newOffsets, newValues); }
@Override public Block apply(Block block) { if (elementCoercer == null) { return block; } ColumnarArray arrayBlock = toColumnarArray(block); Block elementsBlock = elementCoercer.apply(arrayBlock.getElementsBlock()); boolean[] valueIsNull = new boolean[arrayBlock.getPositionCount()]; int[] offsets = new int[arrayBlock.getPositionCount() + 1]; for (int i = 0; i < arrayBlock.getPositionCount(); i++) { valueIsNull[i] = arrayBlock.isNull(i); offsets[i + 1] = offsets[i] + arrayBlock.getLength(i); } return ArrayBlock.fromElementBlock(arrayBlock.getPositionCount(), Optional.of(valueIsNull), offsets, elementsBlock); } }
@Override public Block copyRegion(int position, int length) { int positionCount = getPositionCount(); checkValidRegion(positionCount, position, length); int startValueOffset = getOffset(position); int endValueOffset = getOffset(position + length); Block newValues = getRawElementBlock().copyRegion(startValueOffset, endValueOffset - startValueOffset); int[] newOffsets = compactOffsets(getOffsets(), position + getOffsetBase(), length); boolean[] valueIsNull = getValueIsNull(); boolean[] newValueIsNull = valueIsNull == null ? null : compactArray(valueIsNull, position + getOffsetBase(), length); if (newValues == getRawElementBlock() && newOffsets == getOffsets() && newValueIsNull == valueIsNull) { return this; } return createArrayBlockInternal(0, length, newValueIsNull, newOffsets, newValues); }
@TypeParameter("T") @SqlType("array(array(T))") public static Block ngrams(@SqlType("array(T)") Block array, @SqlType(INTEGER) long n) { checkCondition(n > 0, INVALID_FUNCTION_ARGUMENT, "N must be positive"); // n should not be larger than the array length int elementsPerRecord = toIntExact(min(array.getPositionCount(), n)); int totalRecords = array.getPositionCount() - elementsPerRecord + 1; int[] ids = new int[totalRecords * elementsPerRecord]; int[] offset = new int[totalRecords + 1]; for (int recordIndex = 0; recordIndex < totalRecords; recordIndex++) { for (int elementIndex = 0; elementIndex < elementsPerRecord; elementIndex++) { ids[recordIndex * elementsPerRecord + elementIndex] = recordIndex + elementIndex; } offset[recordIndex + 1] = (recordIndex + 1) * elementsPerRecord; } return ArrayBlock.fromElementBlock(totalRecords, Optional.empty(), offset, array.getPositions(ids, 0, totalRecords * elementsPerRecord)); } }