/** * Clones the header followed by the on-disk (compressed/encoded/encrypted) data. This is * needed for storing packed blocks in the block cache. Expects calling semantics identical to * {@link #getUncompressedBufferWithHeader()}. Returns only the header and data, * Does not include checksum data. * * @return Returns a copy of block bytes for caching on write */ private ByteBuffer cloneOnDiskBufferWithHeader() { expectState(State.BLOCK_READY); return ByteBuffer.wrap(onDiskBlockBytesWithHeader.toByteArray()); }
@Test public void testWriteTag() throws IOException { byte[] tags = Bytes.toBytes("---tags---"); int tagOffset = 3; int length = 4; IndividualBytesFieldCell cell = new IndividualBytesFieldCell(Bytes.toBytes("row"), 0, 3, Bytes.toBytes("family"), 0, 6, Bytes.toBytes("qualifier"), 0, 9, 0L, KeyValue.Type.Put, 0, Bytes.toBytes("value"), 0, 5, tags, tagOffset, length); try (ByteArrayOutputStream output = new ByteArrayOutputStream(300)) { cell.write(output, true); byte[] buf = output.toByteArray(); assertEquals(cell.getSerializedSize(true), buf.length); } }
@Test public void testWriteValue() throws IOException { byte[] value = Bytes.toBytes("---value---"); int valueOffset = 3; int valueLength = 5; IndividualBytesFieldCell cell = new IndividualBytesFieldCell(Bytes.toBytes("row"), 0, 3, Bytes.toBytes("family"), 0, 6, Bytes.toBytes("qualifier"), 0, 9, 0L, KeyValue.Type.Put, 0, value, valueOffset, valueLength, Bytes.toBytes("value"), 0, 5); try (ByteArrayOutputStream output = new ByteArrayOutputStream(300)) { cell.write(output, true); byte[] buf = output.toByteArray(); assertEquals(cell.getSerializedSize(true), buf.length); } } }
@Test public void testLegacySerialization() throws IOException { ByteArrayOutputStream data = new ByteArrayOutputStream(); DataOutputStream output = new DataOutputStream(data); output.writeLong(100); output.writeLong(200); TimeRangeTracker tgt = TimeRangeTracker.parseFrom(data.toByteArray()); assertEquals(100, tgt.getMin()); assertEquals(200, tgt.getMax()); }
static ByteBuffer encodeKeyValues(DataBlockEncoding encoding, List<KeyValue> kvs, HFileBlockEncodingContext encodingContext, boolean useOffheapData) throws IOException { DataBlockEncoder encoder = encoding.getEncoder(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(HFILEBLOCK_DUMMY_HEADER); DataOutputStream dos = new DataOutputStream(baos); encoder.startBlockEncoding(encodingContext, dos); for (KeyValue kv : kvs) { encoder.encode(kv, encodingContext, dos); } encoder.endBlockEncoding(encodingContext, dos, baos.getBuffer()); byte[] encodedData = new byte[baos.size() - ENCODED_DATA_OFFSET]; System.arraycopy(baos.toByteArray(), ENCODED_DATA_OFFSET, encodedData, 0, encodedData.length); if (useOffheapData) { ByteBuffer bb = ByteBuffer.allocateDirect(encodedData.length); bb.put(encodedData); bb.rewind(); return bb; } return ByteBuffer.wrap(encodedData); }
/** * Clones the header followed by the uncompressed data, even if using * compression. This is needed for storing uncompressed blocks in the block * cache. Can be called in the "writing" state or the "block ready" state. * Returns only the header and data, does not include checksum data. * * @return Returns a copy of uncompressed block bytes for caching on write */ @VisibleForTesting ByteBuffer cloneUncompressedBufferWithHeader() { expectState(State.BLOCK_READY); byte[] uncompressedBlockBytesWithHeader = baosInMemory.toByteArray(); int numBytes = (int) ChecksumUtil.numBytes( onDiskBlockBytesWithHeader.size(), fileContext.getBytesPerChecksum()); putHeader(uncompressedBlockBytesWithHeader, 0, onDiskBlockBytesWithHeader.size() + numBytes, baosInMemory.size(), onDiskBlockBytesWithHeader.size()); return ByteBuffer.wrap(uncompressedBlockBytesWithHeader); }
private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags) throws IOException { int size; HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, block.getHFileContext()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(block.getDummyHeaderForVersion()); DataOutputStream dos = new DataOutputStream(baos); blockEncoder.startBlockEncoding(context, dos); for (KeyValue kv : kvs) { blockEncoder.encode(kv, context, dos); } blockEncoder.endBlockEncoding(context, dos, baos.getBuffer(), BlockType.DATA); byte[] encodedBytes = baos.toByteArray(); size = encodedBytes.length - block.getDummyHeaderForVersion().length; return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes), HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), -1, block.getHFileContext()); }
private void testEncodersOnDataset(List<KeyValue> kvList, boolean includesMemstoreTS, boolean includesTags) throws IOException { ByteBuffer unencodedDataBuf = RedundantKVGenerator.convertKvToByteBuffer(kvList, includesMemstoreTS); HFileContext fileContext = new HFileContextBuilder().withIncludesMvcc(includesMemstoreTS) .withIncludesTags(includesTags).build(); for (DataBlockEncoding encoding : DataBlockEncoding.values()) { DataBlockEncoder encoder = encoding.getEncoder(); if (encoder == null) { continue; } HFileBlockEncodingContext encodingContext = new HFileBlockDefaultEncodingContext(encoding, HFILEBLOCK_DUMMY_HEADER, fileContext); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(HFILEBLOCK_DUMMY_HEADER); DataOutputStream dos = new DataOutputStream(baos); encoder.startBlockEncoding(encodingContext, dos); for (KeyValue kv : kvList) { encoder.encode(kv, encodingContext, dos); } encoder.endBlockEncoding(encodingContext, dos, baos.getBuffer()); byte[] encodedData = baos.toByteArray(); testAlgorithm(encodedData, unencodedDataBuf, encoder); } }
@Test public void testCompressUncompressTags1() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); KeyValue kv1 = createKVWithTags(2); int tagsLength1 = kv1.getTagsLength(); ByteBuffer ib = ByteBuffer.wrap(kv1.getTagsArray()); context.compressTags(baos, ib, kv1.getTagsOffset(), tagsLength1); KeyValue kv2 = createKVWithTags(3); int tagsLength2 = kv2.getTagsLength(); ib = ByteBuffer.wrap(kv2.getTagsArray()); context.compressTags(baos, ib, kv2.getTagsOffset(), tagsLength2); context.clear(); byte[] dest = new byte[tagsLength1]; ByteBuffer ob = ByteBuffer.wrap(baos.toByteArray()); context.uncompressTags(new SingleByteBuff(ob), dest, 0, tagsLength1); assertTrue(Bytes.equals(kv1.getTagsArray(), kv1.getTagsOffset(), tagsLength1, dest, 0, tagsLength1)); dest = new byte[tagsLength2]; context.uncompressTags(new SingleByteBuff(ob), dest, 0, tagsLength2); assertTrue(Bytes.equals(kv2.getTagsArray(), kv2.getTagsOffset(), tagsLength2, dest, 0, tagsLength2)); }
compressionStream.flush(); compressionStream.finish(); byte[] plaintext = compressedByteStream.toByteArray(); plaintextLength = plaintext.length; in = new ByteArrayInputStream(plaintext);
@Test public void testWriteTag() throws IOException { byte[] tags = Bytes.toBytes("---tags---"); int tagOffset = 3; int length = 4; IndividualBytesFieldCell cell = new IndividualBytesFieldCell(Bytes.toBytes("row"), 0, 3, Bytes.toBytes("family"), 0, 6, Bytes.toBytes("qualifier"), 0, 9, 0L, KeyValue.Type.Put, 0, Bytes.toBytes("value"), 0, 5, tags, tagOffset, length); try (ByteArrayOutputStream output = new ByteArrayOutputStream(300)) { cell.write(output, true); byte[] buf = output.toByteArray(); assertEquals(cell.getSerializedSize(true), buf.length); } }
@Test public void testWriteTag() throws IOException { byte[] tags = Bytes.toBytes("---tags---"); int tagOffset = 3; int length = 4; IndividualBytesFieldCell cell = new IndividualBytesFieldCell(Bytes.toBytes("row"), 0, 3, Bytes.toBytes("family"), 0, 6, Bytes.toBytes("qualifier"), 0, 9, 0L, KeyValue.Type.Put, 0, Bytes.toBytes("value"), 0, 5, tags, tagOffset, length); try (ByteArrayOutputStream output = new ByteArrayOutputStream(300)) { cell.write(output, true); byte[] buf = output.toByteArray(); assertEquals(cell.getSerializedSize(true), buf.length); } }
@Test public void testWriteValue() throws IOException { byte[] value = Bytes.toBytes("---value---"); int valueOffset = 3; int valueLength = 5; IndividualBytesFieldCell cell = new IndividualBytesFieldCell(Bytes.toBytes("row"), 0, 3, Bytes.toBytes("family"), 0, 6, Bytes.toBytes("qualifier"), 0, 9, 0L, KeyValue.Type.Put, 0, value, valueOffset, valueLength, Bytes.toBytes("value"), 0, 5); try (ByteArrayOutputStream output = new ByteArrayOutputStream(300)) { cell.write(output, true); byte[] buf = output.toByteArray(); assertEquals(cell.getSerializedSize(true), buf.length); } } }
@Test public void testWriteValue() throws IOException { byte[] value = Bytes.toBytes("---value---"); int valueOffset = 3; int valueLength = 5; IndividualBytesFieldCell cell = new IndividualBytesFieldCell(Bytes.toBytes("row"), 0, 3, Bytes.toBytes("family"), 0, 6, Bytes.toBytes("qualifier"), 0, 9, 0L, KeyValue.Type.Put, 0, value, valueOffset, valueLength, Bytes.toBytes("value"), 0, 5); try (ByteArrayOutputStream output = new ByteArrayOutputStream(300)) { cell.write(output, true); byte[] buf = output.toByteArray(); assertEquals(cell.getSerializedSize(true), buf.length); } } }
@Test public void testLegacySerialization() throws IOException { ByteArrayOutputStream data = new ByteArrayOutputStream(); DataOutputStream output = new DataOutputStream(data); output.writeLong(100); output.writeLong(200); TimeRangeTracker tgt = TimeRangeTracker.parseFrom(data.toByteArray()); assertEquals(100, tgt.getMin()); assertEquals(200, tgt.getMax()); }
static ByteBuffer encodeKeyValues(DataBlockEncoding encoding, List<KeyValue> kvs, HFileBlockEncodingContext encodingContext, boolean useOffheapData) throws IOException { DataBlockEncoder encoder = encoding.getEncoder(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(HFILEBLOCK_DUMMY_HEADER); DataOutputStream dos = new DataOutputStream(baos); encoder.startBlockEncoding(encodingContext, dos); for (KeyValue kv : kvs) { encoder.encode(kv, encodingContext, dos); } encoder.endBlockEncoding(encodingContext, dos, baos.getBuffer()); byte[] encodedData = new byte[baos.size() - ENCODED_DATA_OFFSET]; System.arraycopy(baos.toByteArray(), ENCODED_DATA_OFFSET, encodedData, 0, encodedData.length); if (useOffheapData) { ByteBuffer bb = ByteBuffer.allocateDirect(encodedData.length); bb.put(encodedData); bb.rewind(); return bb; } return ByteBuffer.wrap(encodedData); }
private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags) throws IOException { int size; HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, block.getHFileContext()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(block.getDummyHeaderForVersion()); DataOutputStream dos = new DataOutputStream(baos); blockEncoder.startBlockEncoding(context, dos); for (KeyValue kv : kvs) { blockEncoder.encode(kv, context, dos); } blockEncoder.endBlockEncoding(context, dos, baos.getBuffer(), BlockType.DATA); byte[] encodedBytes = baos.toByteArray(); size = encodedBytes.length - block.getDummyHeaderForVersion().length; return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes), HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), -1, block.getHFileContext()); }
@Test public void testCompressUncompressTags1() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); KeyValue kv1 = createKVWithTags(2); int tagsLength1 = kv1.getTagsLength(); ByteBuffer ib = ByteBuffer.wrap(kv1.getTagsArray()); context.compressTags(baos, ib, kv1.getTagsOffset(), tagsLength1); KeyValue kv2 = createKVWithTags(3); int tagsLength2 = kv2.getTagsLength(); ib = ByteBuffer.wrap(kv2.getTagsArray()); context.compressTags(baos, ib, kv2.getTagsOffset(), tagsLength2); context.clear(); byte[] dest = new byte[tagsLength1]; ByteBuffer ob = ByteBuffer.wrap(baos.toByteArray()); context.uncompressTags(new SingleByteBuff(ob), dest, 0, tagsLength1); assertTrue(Bytes.equals(kv1.getTagsArray(), kv1.getTagsOffset(), tagsLength1, dest, 0, tagsLength1)); dest = new byte[tagsLength2]; context.uncompressTags(new SingleByteBuff(ob), dest, 0, tagsLength2); assertTrue(Bytes.equals(kv2.getTagsArray(), kv2.getTagsOffset(), tagsLength2, dest, 0, tagsLength2)); }
private void testEncodersOnDataset(List<KeyValue> kvList, boolean includesMemstoreTS, boolean includesTags) throws IOException { ByteBuffer unencodedDataBuf = RedundantKVGenerator.convertKvToByteBuffer(kvList, includesMemstoreTS); HFileContext fileContext = new HFileContextBuilder().withIncludesMvcc(includesMemstoreTS) .withIncludesTags(includesTags).build(); for (DataBlockEncoding encoding : DataBlockEncoding.values()) { DataBlockEncoder encoder = encoding.getEncoder(); if (encoder == null) { continue; } HFileBlockEncodingContext encodingContext = new HFileBlockDefaultEncodingContext(encoding, HFILEBLOCK_DUMMY_HEADER, fileContext); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(HFILEBLOCK_DUMMY_HEADER); DataOutputStream dos = new DataOutputStream(baos); encoder.startBlockEncoding(encodingContext, dos); for (KeyValue kv : kvList) { encoder.encode(kv, encodingContext, dos); } encoder.endBlockEncoding(encodingContext, dos, baos.getBuffer()); byte[] encodedData = baos.toByteArray(); testAlgorithm(encodedData, unencodedDataBuf, encoder); } }
@Test public void testCompressUncompressTags1() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); KeyValue kv1 = createKVWithTags(2); int tagsLength1 = kv1.getTagsLength(); ByteBuffer ib = ByteBuffer.wrap(kv1.getTagsArray()); context.compressTags(baos, ib, kv1.getTagsOffset(), tagsLength1); KeyValue kv2 = createKVWithTags(3); int tagsLength2 = kv2.getTagsLength(); ib = ByteBuffer.wrap(kv2.getTagsArray()); context.compressTags(baos, ib, kv2.getTagsOffset(), tagsLength2); context.clear(); byte[] dest = new byte[tagsLength1]; ByteBuffer ob = ByteBuffer.wrap(baos.toByteArray()); context.uncompressTags(new SingleByteBuff(ob), dest, 0, tagsLength1); assertTrue(Bytes.equals(kv1.getTagsArray(), kv1.getTagsOffset(), tagsLength1, dest, 0, tagsLength1)); dest = new byte[tagsLength2]; context.uncompressTags(new SingleByteBuff(ob), dest, 0, tagsLength2); assertTrue(Bytes.equals(kv2.getTagsArray(), kv2.getTagsOffset(), tagsLength2, dest, 0, tagsLength2)); }