@Override public long getAllocatedSize() { return encoder.getAllocatedSize(); }
@Override public void reset() { encoder = new ByteBasedBitPackingEncoder(bitWidth, packer); }
@Override public long getBufferedSize() { return encoder.getBufferSize(); }
/** * writes an int using the requested number of bits. * accepts only values less than 2^bitWidth * @param value the value to write * @throws IOException if there is an exception while writing */ public void writeInt(int value) throws IOException { input[inputSize] = value; ++ inputSize; if (inputSize == VALUES_WRITTEN_AT_A_TIME) { pack(); if (packedPosition == slabSize) { slabs.add(BytesInput.from(packed)); totalFullSlabSize += slabSize; if (slabSize < bitWidth * MAX_SLAB_SIZE_MULT) { slabSize *= 2; } initPackedSlab(); } } }
@Override public void writeInteger(int v) { try { this.encoder.writeInt(v); } catch (IOException e) { throw new ParquetEncodingException(e); } }
@Override public String memUsageString(String prefix) { return encoder.memUsageString(prefix); }
/** * @param bitWidth the number of bits used to encode an int * @param packer factory for bit packing implementations */ public ByteBasedBitPackingEncoder(int bitWidth, Packer packer) { this.bitWidth = bitWidth; this.inputSize = 0; this.totalFullSlabSize = 0; // must be a multiple of bitWidth this.slabSize = (bitWidth == 0) ? 1 : (bitWidth * INITIAL_SLAB_SIZE_MULT); initPackedSlab(); this.packer = packer.newBytePacker(bitWidth); }
@Override public BytesInput getBytes() { try { return encoder.toBytes(); } catch (IOException e) { throw new ParquetEncodingException(e); } }
/** * @return the bytes representing the packed values * @throws IOException if there is an exception while creating the BytesInput */ public BytesInput toBytes() throws IOException { int packedByteLength = packedPosition + BytesUtils.paddedByteCountFromBits(inputSize * bitWidth); LOG.debug("writing {} bytes", (totalFullSlabSize + packedByteLength)); if (inputSize > 0) { for (int i = inputSize; i < input.length; i++) { input[i] = 0; } pack(); } return concat(concat(slabs), BytesInput.from(packed, 0, packedByteLength)); }
/** * writes an int using the requested number of bits. * accepts only values less than 2^bitWidth * @param value the value to write * @throws IOException if there is an exception while writing */ public void writeInt(int value) throws IOException { input[inputSize] = value; ++ inputSize; if (inputSize == VALUES_WRITTEN_AT_A_TIME) { pack(); if (packedPosition == slabSize) { slabs.add(BytesInput.from(packed)); totalFullSlabSize += slabSize; if (slabSize < bitWidth * MAX_SLAB_SIZE_MULT) { slabSize *= 2; } initPackedSlab(); } } }
@Override public void writeInteger(int v) { try { this.encoder.writeInt(v); } catch (IOException e) { throw new ParquetEncodingException(e); } }
@Override public String memUsageString(String prefix) { return encoder.memUsageString(prefix); }
/** * @param bitWidth the number of bits used to encode an int * @param packer factory for bit packing implementations */ public ByteBasedBitPackingEncoder(int bitWidth, Packer packer) { this.bitWidth = bitWidth; this.inputSize = 0; this.totalFullSlabSize = 0; // must be a multiple of bitWidth this.slabSize = (bitWidth == 0) ? 1 : (bitWidth * INITIAL_SLAB_SIZE_MULT); initPackedSlab(); this.packer = packer.newBytePacker(bitWidth); }
@Override public BytesInput getBytes() { try { return encoder.toBytes(); } catch (IOException e) { throw new ParquetEncodingException(e); } }
/** * @return the bytes representing the packed values * @throws IOException if there is an exception while creating the BytesInput */ public BytesInput toBytes() throws IOException { int packedByteLength = packedPosition + BytesUtils.paddedByteCountFromBits(inputSize * bitWidth); LOG.debug("writing {} bytes", (totalFullSlabSize + packedByteLength)); if (inputSize > 0) { for (int i = inputSize; i < input.length; i++) { input[i] = 0; } pack(); } return concat(concat(slabs), BytesInput.from(packed, 0, packedByteLength)); }
@Override public long getAllocatedSize() { return encoder.getAllocatedSize(); }
@Override public void reset() { encoder = new ByteBasedBitPackingEncoder(bitWidth, packer); }
@Override public long getBufferedSize() { return encoder.getBufferSize(); }
public String memUsageString(String prefix) { return String.format("%s ByteBitPacking %d slabs, %d bytes", prefix, slabs.size(), getAllocatedSize()); }
public ByteBitPackingValuesWriter(int bound, Packer packer) { this.packer = packer; this.bitWidth = BytesUtils.getWidthFromMaxInt(bound); this.encoder = new ByteBasedBitPackingEncoder(bitWidth, packer); }