private static BlockBuilder createBlockBuilder() { return new VariableWidthBlockBuilder(null, 100, 1); }
private Block getSliceKeysBlockTemplate(List<DwrfSequenceEncoding> sequenceEncodings) { int bytes = 0; for (DwrfSequenceEncoding sequenceEncoding : sequenceEncodings) { bytes += sequenceEncoding.getKey().getBytesKey().size(); } VariableWidthBlockBuilder builder = new VariableWidthBlockBuilder(null, sequenceEncodings.size(), bytes); for (DwrfSequenceEncoding sequenceEncoding : sequenceEncodings) { Slice key = Slices.wrappedBuffer(sequenceEncoding.getKey().getBytesKey().toByteArray()); builder.writeBytes(key, 0, key.length()); builder.closeEntry(); } return builder.build(); }
@Override public BlockBuilder closeEntry() { entryAdded(currentEntrySize, false); currentEntrySize = 0; return this; }
@Override public long getRegionSizeInBytes(int positionOffset, int length) { int positionCount = getPositionCount(); checkValidRegion(positionCount, positionOffset, length); long arraysSizeInBytes = (Integer.BYTES + Byte.BYTES) * (long) length; return getOffset(positionOffset + length) - getOffset(positionOffset) + arraysSizeInBytes; }
@Override public BlockBuilder newBlockBuilderLike(BlockBuilderStatus blockBuilderStatus) { int currentSizeInBytes = positions == 0 ? positions : (getOffset(positions) - getOffset(0)); return new VariableWidthBlockBuilder(blockBuilderStatus, calculateBlockResetSize(positions), calculateBlockResetBytes(currentSizeInBytes)); }
@Test private void testGetSizeInBytes() { int numEntries = 1000; VarcharType unboundedVarcharType = createUnboundedVarcharType(); VariableWidthBlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, numEntries, 20 * numEntries); for (int i = 0; i < numEntries; i++) { unboundedVarcharType.writeString(blockBuilder, String.valueOf(ThreadLocalRandom.current().nextLong())); } Block block = blockBuilder.build(); List<Block> splitQuarter = splitBlock(block, 4); long sizeInBytes = block.getSizeInBytes(); long quarter1size = splitQuarter.get(0).getSizeInBytes(); long quarter2size = splitQuarter.get(1).getSizeInBytes(); long quarter3size = splitQuarter.get(2).getSizeInBytes(); long quarter4size = splitQuarter.get(3).getSizeInBytes(); double expectedQuarterSizeMin = sizeInBytes * 0.2; double expectedQuarterSizeMax = sizeInBytes * 0.3; assertTrue(quarter1size > expectedQuarterSizeMin && quarter1size < expectedQuarterSizeMax, format("quarter1size is %s, should be between %s and %s", quarter1size, expectedQuarterSizeMin, expectedQuarterSizeMax)); assertTrue(quarter2size > expectedQuarterSizeMin && quarter2size < expectedQuarterSizeMax, format("quarter2size is %s, should be between %s and %s", quarter2size, expectedQuarterSizeMin, expectedQuarterSizeMax)); assertTrue(quarter3size > expectedQuarterSizeMin && quarter3size < expectedQuarterSizeMax, format("quarter3size is %s, should be between %s and %s", quarter3size, expectedQuarterSizeMin, expectedQuarterSizeMax)); assertTrue(quarter4size > expectedQuarterSizeMin && quarter4size < expectedQuarterSizeMax, format("quarter4size is %s, should be between %s and %s", quarter4size, expectedQuarterSizeMin, expectedQuarterSizeMax)); assertEquals(quarter1size + quarter2size + quarter3size + quarter4size, sizeInBytes); }
@Override public Block copyPositions(int[] positions, int offset, int length) { checkArrayRange(positions, offset, length); int finalLength = 0; for (int i = offset; i < offset + length; i++) { finalLength += getSliceLength(positions[i]); } SliceOutput newSlice = Slices.allocate(finalLength).getOutput(); int[] newOffsets = new int[length + 1]; boolean[] newValueIsNull = null; if (hasNullValue) { newValueIsNull = new boolean[length]; } for (int i = 0; i < length; i++) { int position = positions[offset + i]; if (isEntryNull(position)) { newValueIsNull[i] = true; } else { newSlice.writeBytes(sliceOutput.getUnderlyingSlice(), getPositionOffset(position), getSliceLength(position)); } newOffsets[i + 1] = newSlice.size(); } return new VariableWidthBlock(0, length, newSlice.slice(), newOffsets, newValueIsNull); }
@Override public Block getRegion(int positionOffset, int length) { int positionCount = getPositionCount(); checkValidRegion(positionCount, positionOffset, length); return new VariableWidthBlock(positionOffset, length, sliceOutput.slice(), offsets, hasNullValue ? valueIsNull : null); }
@Test private void testGetSizeInBytes() { int numEntries = 1000; VarcharType unboundedVarcharType = createUnboundedVarcharType(); VariableWidthBlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, numEntries, 20 * numEntries); for (int i = 0; i < numEntries; i++) { unboundedVarcharType.writeString(blockBuilder, String.valueOf(ThreadLocalRandom.current().nextLong())); } Block block = blockBuilder.build(); List<Block> splitQuarter = splitBlock(block, 4); long sizeInBytes = block.getSizeInBytes(); long quarter1size = splitQuarter.get(0).getSizeInBytes(); long quarter2size = splitQuarter.get(1).getSizeInBytes(); long quarter3size = splitQuarter.get(2).getSizeInBytes(); long quarter4size = splitQuarter.get(3).getSizeInBytes(); double expectedQuarterSizeMin = sizeInBytes * 0.2; double expectedQuarterSizeMax = sizeInBytes * 0.3; assertTrue(quarter1size > expectedQuarterSizeMin && quarter1size < expectedQuarterSizeMax, format("quarter1size is %s, should be between %s and %s", quarter1size, expectedQuarterSizeMin, expectedQuarterSizeMax)); assertTrue(quarter2size > expectedQuarterSizeMin && quarter2size < expectedQuarterSizeMax, format("quarter2size is %s, should be between %s and %s", quarter2size, expectedQuarterSizeMin, expectedQuarterSizeMax)); assertTrue(quarter3size > expectedQuarterSizeMin && quarter3size < expectedQuarterSizeMax, format("quarter3size is %s, should be between %s and %s", quarter3size, expectedQuarterSizeMin, expectedQuarterSizeMax)); assertTrue(quarter4size > expectedQuarterSizeMin && quarter4size < expectedQuarterSizeMax, format("quarter4size is %s, should be between %s and %s", quarter4size, expectedQuarterSizeMin, expectedQuarterSizeMax)); assertEquals(quarter1size + quarter2size + quarter3size + quarter4size, sizeInBytes); }
@Override public long getPositionsSizeInBytes(boolean[] positions) { checkValidPositions(positions, getPositionCount()); long sizeInBytes = 0; int usedPositionCount = 0; for (int i = 0; i < positions.length; ++i) { if (positions[i]) { usedPositionCount++; sizeInBytes += getOffset(i + 1) - getOffset(i); } } return sizeInBytes + (Integer.BYTES + Byte.BYTES) * (long) usedPositionCount; }
@Override protected int getPositionOffset(int position) { checkValidPosition(position, positions); return getOffset(position); }
@Override public Block copyRegion(int positionOffset, int length) { int positionCount = getPositionCount(); checkValidRegion(positionCount, positionOffset, length); int[] newOffsets = compactOffsets(offsets, positionOffset, length); boolean[] newValueIsNull = null; if (hasNullValue) { newValueIsNull = compactArray(valueIsNull, positionOffset, length); } Slice slice = compactSlice(sliceOutput.getUnderlyingSlice(), offsets[positionOffset], newOffsets[length]); return new VariableWidthBlock(0, length, slice, newOffsets, newValueIsNull); }
private static BlockBuilder createBlockBuilder() { return new VariableWidthBlockBuilder(null, 1, 1); }
private Block getSliceKeysBlockTemplate(List<DwrfSequenceEncoding> sequenceEncodings) { int bytes = 0; for (DwrfSequenceEncoding sequenceEncoding : sequenceEncodings) { bytes += sequenceEncoding.getKey().getBytesKey().size(); } VariableWidthBlockBuilder builder = new VariableWidthBlockBuilder(null, sequenceEncodings.size(), bytes); for (DwrfSequenceEncoding sequenceEncoding : sequenceEncodings) { Slice key = Slices.wrappedBuffer(sequenceEncoding.getKey().getBytesKey().toByteArray()); builder.writeBytes(key, 0, key.length()); builder.closeEntry(); } return builder.build(); }
@Override public BlockBuilder appendNull() { if (currentEntrySize > 0) { throw new IllegalStateException("Current entry must be closed before a null can be written"); } hasNullValue = true; entryAdded(0, true); return this; }
private static BlockBuilder createBlockBuilder() { return new VariableWidthBlockBuilder(null, 100, 1); }
private static BlockBuilder createBlockBuilder() { return new VariableWidthBlockBuilder(null, 1, 1); }
@Override public BlockBuilder createBlockBuilder(BlockBuilderStatus blockBuilderStatus, int expectedEntries, int expectedBytesPerEntry) { int maxBlockSizeInBytes; if (blockBuilderStatus == null) { maxBlockSizeInBytes = PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES; } else { maxBlockSizeInBytes = blockBuilderStatus.getMaxPageSizeInBytes(); } // it is guaranteed Math.min will not overflow; safe to cast int expectedBytes = (int) min((long) expectedEntries * expectedBytesPerEntry, maxBlockSizeInBytes); return new VariableWidthBlockBuilder( blockBuilderStatus, expectedBytesPerEntry == 0 ? expectedEntries : Math.min(expectedEntries, maxBlockSizeInBytes / expectedBytesPerEntry), expectedBytes); }
private static BlockBuilder createBlockBuilderWithValues(Slice[] expectedValues) { BlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, expectedValues.length, 32 * expectedValues.length); return writeValues(expectedValues, blockBuilder); }