static Slice concatSlice(Slice... slices) { int totalLength = Arrays.stream(slices) .mapToInt(Slice::length) .sum(); Slice value = Slices.allocate(totalLength); SliceOutput output = value.getOutput(); for (Slice slice : slices) { output.writeBytes(slice); } return value; } }
@Test public void test() throws IOException { List<List<Slice>> groups = new ArrayList<>(); for (int groupIndex = 0; groupIndex < 3; groupIndex++) { List<Slice> group = new ArrayList<>(); for (int i = 0; i < 1000; i++) { Slice value = Slices.allocate(8); SliceOutput output = value.getOutput(); output.writeInt(groupIndex); output.writeInt(i); group.add(value); } groups.add(group); } testWriteValue(groups); }
@Override public Block copyPositions(int[] positions, int offset, int length) { checkArrayRange(positions, offset, length); SliceOutput newSlice = Slices.allocate(length * fixedSize).getOutput(); SliceOutput newValueIsNull = null; if (valueIsNull != null) { newValueIsNull = Slices.allocate(length).getOutput(); } for (int i = offset; i < offset + length; ++i) { int position = positions[i]; checkValidPosition(position, positionCount); newSlice.writeBytes(slice, position * fixedSize, fixedSize); if (valueIsNull != null) { newValueIsNull.writeByte(valueIsNull.getByte(position)); } } return new FixedWidthBlock(fixedSize, length, newSlice.slice(), newValueIsNull == null ? null : newValueIsNull.slice()); }
@Override public Block copyPositions(int[] positions, int offset, int length) { checkArrayRange(positions, offset, length); SliceOutput newSlice = Slices.allocate(length * fixedSize).getOutput(); SliceOutput newValueIsNull = null; if (hasNullValue) { newValueIsNull = Slices.allocate(length).getOutput(); } for (int i = offset; i < offset + length; ++i) { int position = positions[i]; checkValidPosition(position, positionCount); if (hasNullValue) { newValueIsNull.appendByte(valueIsNull.getUnderlyingSlice().getByte(position)); } newSlice.writeBytes(getRawSlice(), position * fixedSize, fixedSize); } return new FixedWidthBlock(fixedSize, length, newSlice.slice(), newValueIsNull == null ? null : newValueIsNull.slice()); }
@Override public void serialize(LongDecimalWithOverflowState state, BlockBuilder out) { if (state.getLongDecimal() == null) { out.appendNull(); } else { Slice slice = Slices.allocate(Long.BYTES + UnscaledDecimal128Arithmetic.UNSCALED_DECIMAL_128_SLICE_LENGTH); SliceOutput output = slice.getOutput(); output.writeLong(state.getOverflow()); output.writeBytes(state.getLongDecimal()); VARBINARY.writeSlice(out, slice); } }
@Override public byte[] getSerializedData() { // Serialization format is (<key:int><min:double><max:double>)* SliceOutput output = Slices.allocate((SizeOf.SIZE_OF_INT + 2 * SizeOf.SIZE_OF_DOUBLE) * mins.size()).getOutput(); for (int key : mins.keySet()) { output.appendInt(key); output.appendDouble(mins.get(key)); output.appendDouble(maxs.get(key)); } return output.slice().getBytes(); }
@Override public void serialize(DigestAndPercentileArrayState state, BlockBuilder out) { if (state.getDigest() == null) { out.appendNull(); } else { Slice digest = state.getDigest().serialize(); SliceOutput output = Slices.allocate( SIZE_OF_INT + // number of percentiles state.getPercentiles().size() * SIZE_OF_DOUBLE + // percentiles SIZE_OF_INT + // digest length digest.length()) // digest .getOutput(); // write percentiles List<Double> percentiles = state.getPercentiles(); output.appendInt(percentiles.size()); for (double percentile : percentiles) { output.appendDouble(percentile); } output.appendInt(digest.length()); output.appendBytes(digest); VARBINARY.writeSlice(out, output.slice()); } }
@Override public void serialize(LongDecimalWithOverflowAndLongState state, BlockBuilder out) { if (state.getLongDecimal() == null) { out.appendNull(); } else { Slice slice = Slices.allocate(Long.BYTES + Long.BYTES + UnscaledDecimal128Arithmetic.UNSCALED_DECIMAL_128_SLICE_LENGTH); SliceOutput output = slice.getOutput(); output.writeLong(state.getLong()); output.writeLong(state.getOverflow()); output.writeBytes(state.getLongDecimal()); VARBINARY.writeSlice(out, slice); } }
public Slice serialize() { compact(); int requiredBytes = SizeOf.SIZE_OF_BYTE + // format SizeOf.SIZE_OF_INT + // max buckets SizeOf.SIZE_OF_INT + // entry count SizeOf.SIZE_OF_DOUBLE * nextIndex + // values SizeOf.SIZE_OF_DOUBLE * nextIndex; // weights return Slices.allocate(requiredBytes) .getOutput() .appendByte(FORMAT_TAG) .appendInt(maxBuckets) .appendInt(nextIndex) .appendBytes(Slices.wrappedDoubleArray(values, 0, nextIndex)) .appendBytes(Slices.wrappedDoubleArray(weights, 0, nextIndex)) .getUnderlyingSlice(); }
@Override public Block copyPositions(int[] positions, int offset, int length) { checkArrayRange(positions, offset, length); int finalLength = 0; for (int i = offset; i < offset + length; i++) { finalLength += getSliceLength(positions[i]); } SliceOutput newSlice = Slices.allocate(finalLength).getOutput(); int[] newOffsets = new int[length + 1]; boolean[] newValueIsNull = null; if (valueIsNull != null) { newValueIsNull = new boolean[length]; } for (int i = 0; i < length; i++) { int position = positions[offset + i]; if (!isEntryNull(position)) { newSlice.writeBytes(slice, getPositionOffset(position), getSliceLength(position)); } else if (newValueIsNull != null) { newValueIsNull[i] = true; } newOffsets[i + 1] = newSlice.size(); } return new VariableWidthBlock(0, length, newSlice.slice(), newOffsets, newValueIsNull); }
@Override public void serialize(DigestAndPercentileState state, BlockBuilder out) { if (state.getDigest() == null) { out.appendNull(); } else { Slice serialized = state.getDigest().serialize(); SliceOutput output = Slices.allocate(SIZE_OF_DOUBLE + SIZE_OF_INT + serialized.length()).getOutput(); output.appendDouble(state.getPercentile()); output.appendInt(serialized.length()); output.appendBytes(serialized); VARBINARY.writeSlice(out, output.slice()); } }
@Override public Block copyPositions(int[] positions, int offset, int length) { checkArrayRange(positions, offset, length); int finalLength = 0; for (int i = offset; i < offset + length; i++) { finalLength += getSliceLength(positions[i]); } SliceOutput newSlice = Slices.allocate(finalLength).getOutput(); int[] newOffsets = new int[length + 1]; boolean[] newValueIsNull = null; if (hasNullValue) { newValueIsNull = new boolean[length]; } for (int i = 0; i < length; i++) { int position = positions[offset + i]; if (isEntryNull(position)) { newValueIsNull[i] = true; } else { newSlice.writeBytes(sliceOutput.getUnderlyingSlice(), getPositionOffset(position), getSliceLength(position)); } newOffsets[i + 1] = newSlice.size(); } return new VariableWidthBlock(0, length, newSlice.slice(), newOffsets, newValueIsNull); }
@SuppressWarnings("AssignmentToForLoopParameter") private static ColumnData unescape(ColumnData columnData, byte escapeByte) { Slice slice = columnData.getSlice(); // does slice contain escape byte if (slice.indexOfByte(escapeByte) < 0) { return columnData; } Slice newSlice = Slices.allocate(slice.length()); SliceOutput output = newSlice.getOutput(); int[] newOffsets = new int[columnData.rowCount() + 1]; for (int row = 0; row < columnData.rowCount(); row++) { int offset = columnData.getOffset(row); int length = columnData.getLength(row); for (int i = 0; i < length; i++) { byte value = slice.getByte(offset + i); if (value == escapeByte && i + 1 < length) { // read byte after escape i++; value = slice.getByte(offset + i); } output.write(value); } newOffsets[row + 1] = output.size(); } return new ColumnData(newOffsets, output.slice()); }
@Test public void testReset() throws Exception { assertReset(new DynamicSliceOutput(1)); assertReset(Slices.allocate(50).getOutput()); }
@Test public void testReset() throws Exception { assertReset(new DynamicSliceOutput(1)); assertReset(Slices.allocate(50).getOutput()); }
protected void testSliceInput(SliceInputTester tester) { Slice slice = Slices.allocate((BUFFER_SIZE * 3) + 10); SliceOutput output = slice.getOutput(); for (int i = 0; i < slice.length() / tester.valueSize(); i++) { tester.loadValue(output, i); } testReadForward(tester, slice); testReadReverse(tester, slice); testReadOffEnd(tester, slice); }
protected void testSliceInput(SliceInputTester tester) { Slice slice = Slices.allocate((BUFFER_SIZE * 3) + 10); SliceOutput output = slice.getOutput(); for (int i = 0; i < slice.length() / tester.valueSize(); i++) { tester.loadValue(output, i); } testReadForward(tester, slice); testReadReverse(tester, slice); testReadOffEnd(tester, slice); }