protected Cell getFirstKeyCellInBlock(HFileBlock curBlock) { ByteBuff buffer = curBlock.getBufferWithoutHeader(); // It is safe to manipulate this buffer because we own the buffer object. buffer.rewind(); int klen = buffer.getInt(); buffer.skip(Bytes.SIZEOF_INT);// Skip value len part ByteBuffer keyBuff = buffer.asSubByteBuffer(klen); if (keyBuff.hasArray()) { return new KeyValue.KeyOnlyKeyValue(keyBuff.array(), keyBuff.arrayOffset() + keyBuff.position(), klen); } else { return new ByteBufferKeyOnlyKeyValue(keyBuff, keyBuff.position(), klen); } }
Math.min(32, buf.limit() - buf.arrayOffset() - headerSize())); } else { ByteBuff bufWithoutHeader = getBufferWithoutHeader(); byte[] dataBeginBytes = new byte[Math.min(32, bufWithoutHeader.limit() - bufWithoutHeader.position())];
/** * Updates the current block to be the given {@link HFileBlock}. Seeks to * the the first key/value pair. * * @param newBlock the block to make current */ protected void updateCurrentBlock(HFileBlock newBlock) throws IOException { // Set the active block on the reader // sanity check if (newBlock.getBlockType() != BlockType.DATA) { throw new IllegalStateException("ScannerV2 works only on data " + "blocks, got " + newBlock.getBlockType() + "; " + "fileName=" + reader.getName() + ", " + "dataBlockEncoder=" + reader.getDataBlockEncoding() + ", " + "isCompaction=" + isCompaction); } updateCurrBlockRef(newBlock); blockBuffer = newBlock.getBufferWithoutHeader(); readKeyValueLen(); blockFetches.incrementAndGet(); // Reset the next indexed key this.nextIndexedKey = null; }
BlockType.LEAF_INDEX, null); try { ByteBuff b = midLeafBlock.getBufferWithoutHeader(); int numDataBlocks = b.getIntAfterPosition(0); int keyRelOffset = b.getIntAfterPosition(Bytes.SIZEOF_INT * (midKeyEntry + 1));
private void readNumMetablocks(Reader reader, int n) throws IOException { for (int i = 0; i < n; i++) { ByteBuff actual = reader.getMetaBlock("HFileMeta" + i, false).getBufferWithoutHeader(); ByteBuffer expected = ByteBuffer.wrap(("something to test" + i).getBytes()); assertEquals( "failed to match metadata", Bytes.toStringBinary(expected), Bytes.toStringBinary(actual.array(), actual.arrayOffset() + actual.position(), actual.capacity())); } }
ByteBuff buffer = block.getBufferWithoutHeader(); index = locateNonRootIndexEntry(buffer, key, comparator); if (index == -1) {
} else { bloomBlock = reader.getMetaBlock(HFile.BLOOM_FILTER_DATA_KEY, true); bloom = bloomBlock.getBufferWithoutHeader(); shouldCheckBloom = bloom != null;
/** * Retrieves the decompressed/decrypted view of this block. An encoded block remains in its * encoded structure. Internal structures are shared between instances where applicable. */ HFileBlock unpack(HFileContext fileContext, FSReader reader) throws IOException { if (!fileContext.isCompressedOrEncrypted()) { // TODO: cannot use our own fileContext here because HFileBlock(ByteBuffer, boolean), // which is used for block serialization to L2 cache, does not preserve encoding and // encryption details. return this; } HFileBlock unpacked = new HFileBlock(this); unpacked.allocateBuffer(); // allocates space for the decompressed block HFileBlockDecodingContext ctx = blockType == BlockType.ENCODED_DATA ? reader.getBlockDecodingContext() : reader.getDefaultBlockDecodingContext(); ByteBuff dup = this.buf.duplicate(); dup.position(this.headerSize()); dup = dup.slice(); ctx.prepareDecoding(unpacked.getOnDiskSizeWithoutHeader(), unpacked.getUncompressedSizeWithoutHeader(), unpacked.getBufferWithoutHeader(), dup); return unpacked; }
is, totalSize, (HFileSystem) fs, path, meta); HFileBlock b = hbr.readBlockData(0, -1, false, false); ByteBuff data = b.getBufferWithoutHeader(); for (int i = 0; i < 1000; i++) { assertEquals(i, data.getInt());
if (cc.shouldCacheCompressed(block.getBlockType().getCategory())) { assertFalse("found an unpacked block, block=" + block + ", block buffer capacity=" + block.getBufferWithoutHeader().capacity(), block.isUnpacked());
packedHeapsize < blockUnpacked.heapSize()); ByteBuff actualBuffer = blockUnpacked.getBufferWithoutHeader(); if (encoding != DataBlockEncoding.NONE) {
.unpack(context, blockReader); assertEquals(BlockType.DATA, block.getBlockType()); ByteBuff buf = block.getBufferWithoutHeader(); int keyLen = -1; while (buf.hasRemaining()) { assertEquals(BlockType.META, block.getBlockType()); Text t = new Text(); ByteBuff buf = block.getBufferWithoutHeader(); if (Writables.getWritable(buf.array(), buf.arrayOffset(), buf.limit(), t) == null) { throw new IOException("Failed to deserialize block " + this +
b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes()); ByteBuff bb = b.unpack(meta, hbr).getBufferWithoutHeader(); // read back data DataInputStream in = new DataInputStream( new ByteArrayInputStream( b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes()); bb = b.getBufferWithoutHeader(); // read back data in = new DataInputStream(new ByteArrayInputStream( bb.array(), bb.arrayOffset(), bb.limit()));
@Override protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) { ByteBuffer buffer = curBlock.getBufferWithoutHeader(); // It is safe to manipulate this buffer because we own the buffer object. buffer.rewind(); int klen = buffer.getInt(); buffer.getInt(); ByteBuffer keyBuff = buffer.slice(); keyBuff.limit(klen); keyBuff.rewind(); return keyBuff; }
@Override protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) { ByteBuffer buffer = curBlock.getBufferWithoutHeader(); // It is safe to manipulate this buffer because we own the buffer object. buffer.rewind(); int klen = buffer.getInt(); buffer.getInt(); ByteBuffer keyBuff = buffer.slice(); keyBuff.limit(klen); keyBuff.rewind(); return keyBuff; }
/** * Updates the current block to be the given {@link HFileBlock}. Seeks to * the the first key/value pair. * * @param newBlock the block to make current */ protected void updateCurrBlock(HFileBlock newBlock) { block = newBlock; // sanity check if (block.getBlockType() != BlockType.DATA) { throw new IllegalStateException("ScannerV2 works only on data " + "blocks, got " + block.getBlockType() + "; " + "fileName=" + reader.name + ", " + "dataBlockEncoder=" + reader.dataBlockEncoder + ", " + "isCompaction=" + isCompaction); } blockBuffer = block.getBufferWithoutHeader(); readKeyValueLen(); blockFetches.incrementAndGet(); // Reset the next indexed key this.nextIndexedKey = null; }
/** * Updates the current block to be the given {@link HFileBlock}. Seeks to * the the first key/value pair. * * @param newBlock the block to make current */ private void updateCurrBlock(HFileBlock newBlock) { block = newBlock; // sanity check if (block.getBlockType() != BlockType.DATA) { throw new IllegalStateException("ScannerV2 works only on data " + "blocks, got " + block.getBlockType() + "; " + "fileName=" + reader.name + ", " + "dataBlockEncoder=" + reader.dataBlockEncoder + ", " + "isCompaction=" + isCompaction); } blockBuffer = block.getBufferWithoutHeader(); readKeyValueLen(); blockFetches++; // Reset the next indexed key this.nextIndexedKey = null; }
private void readNumMetablocks(Reader reader, int n) throws IOException { for (int i = 0; i < n; i++) { ByteBuff actual = reader.getMetaBlock("HFileMeta" + i, false).getBufferWithoutHeader(); ByteBuffer expected = ByteBuffer.wrap(("something to test" + i).getBytes()); assertEquals( "failed to match metadata", Bytes.toStringBinary(expected), Bytes.toStringBinary(actual.array(), actual.arrayOffset() + actual.position(), actual.capacity())); } }
private HFileBlock encodeDataBlock(HFileBlock block, DataBlockEncoding algo, boolean includesMemstoreTS) { ByteBuffer compressedBuffer = encodeBufferToHFileBlockBuffer( block.getBufferWithoutHeader(), algo, includesMemstoreTS, block.getDummyHeaderForVersion()); int sizeWithoutHeader = compressedBuffer.limit() - block.headerSize(); HFileBlock encodedBlock = new HFileBlock(BlockType.ENCODED_DATA, block.getOnDiskSizeWithoutHeader(), sizeWithoutHeader, block.getPrevBlockOffset(), compressedBuffer, HFileBlock.FILL_HEADER, block.getOffset(), includesMemstoreTS, block.getMinorVersion(), block.getBytesPerChecksum(), block.getChecksumType(), block.getOnDiskDataSizeWithHeader()); block.passSchemaMetricsTo(encodedBlock); return encodedBlock; }
is, totalSize, (HFileSystem) fs, path, meta); HFileBlock b = hbr.readBlockData(0, -1, false, false); ByteBuff data = b.getBufferWithoutHeader(); for (int i = 0; i < 1000; i++) { assertEquals(i, data.getInt());