private ByteBuff getEncodedBuffer(HFileBlock newBlock) { ByteBuff origBlock = newBlock.getBufferReadOnly(); int pos = newBlock.headerSize() + DataBlockEncoding.ID_SIZE; origBlock.position(pos); origBlock .limit(pos + newBlock.getUncompressedSizeWithoutHeader() - DataBlockEncoding.ID_SIZE); return origBlock.slice(); }
/** * Returns a buffer that does not include the header or checksum. * * @return the buffer with header skipped and checksum omitted. */ public ByteBuff getBufferWithoutHeader() { ByteBuff dup = getBufferReadOnly(); // Now set it up so Buffer spans content only -- no header or no checksums. return dup.position(headerSize()).limit(buf.limit() - totalChecksumBytes()).slice(); }
@Override public boolean contains(byte[] key, int keyOffset, int keyLength, ByteBuff bloom) { int block = index.rootBlockContainingKey(key, keyOffset, keyLength); if (block < 0) { return false; // This key is not in the file. } boolean result; HFileBlock bloomBlock = getBloomBlock(block); try { ByteBuff bloomBuf = bloomBlock.getBufferReadOnly(); result = BloomFilterUtil.contains(key, keyOffset, keyLength, bloomBuf, bloomBlock.headerSize(), bloomBlock.getUncompressedSizeWithoutHeader(), hash, hashCount); } finally { // After the use return back the block if it was served from a cache. reader.returnBlock(bloomBlock); } if (numPositivesPerChunk != null && result) { // Update statistics. Only used in unit tests. ++numPositivesPerChunk[block]; } return result; }
@Override public boolean contains(Cell keyCell, ByteBuff bloom, BloomType type) { int block = index.rootBlockContainingKey(keyCell); if (block < 0) { return false; // This key is not in the file. } boolean result; HFileBlock bloomBlock = getBloomBlock(block); try { ByteBuff bloomBuf = bloomBlock.getBufferReadOnly(); result = BloomFilterUtil.contains(keyCell, bloomBuf, bloomBlock.headerSize(), bloomBlock.getUncompressedSizeWithoutHeader(), hash, hashCount, type); } finally { // After the use return back the block if it was served from a cache. reader.returnBlock(bloomBlock); } if (numPositivesPerChunk != null && result) { // Update statistics. Only used in unit tests. ++numPositivesPerChunk[block]; } return result; }
ByteBuff sliceBuf = block.getBufferReadOnly(); ByteBuffer metadata = block.getMetaData(); if (LOG.isTraceEnabled()) {
private void testEncodingWithCacheInternals(boolean useTag) throws IOException { List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag); HFileBlock block = getSampleHFileBlock(kvs, useTag); HFileBlock cacheBlock = createBlockOnDisk(kvs, block, useTag); LruBlockCache blockCache = new LruBlockCache(8 * 1024 * 1024, 32 * 1024); BlockCacheKey cacheKey = new BlockCacheKey("test", 0); blockCache.cacheBlock(cacheKey, cacheBlock); HeapSize heapSize = blockCache.getBlock(cacheKey, false, false, true); assertTrue(heapSize instanceof HFileBlock); HFileBlock returnedBlock = (HFileBlock) heapSize; if (blockEncoder.getDataBlockEncoding() == DataBlockEncoding.NONE) { assertEquals(block.getBufferReadOnly(), returnedBlock.getBufferReadOnly()); } else { if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) { System.out.println(blockEncoder); } assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType()); } }
ByteBuff bufRead = b.getBufferReadOnly(); ByteBuffer bufExpected = expectedContents.get(i); boolean bytesAreCorrect = Bytes.compareTo(bufRead.array(),
if (block.getBlockType() != BlockType.LEAF_INDEX) return; ByteBuff b = block.getBufferReadOnly(); int n = b.getIntAfterPosition(0);
private ByteBuffer getEncodedBuffer(HFileBlock newBlock) { ByteBuffer origBlock = newBlock.getBufferReadOnly(); ByteBuffer encodedBlock = ByteBuffer.wrap(origBlock.array(), origBlock.arrayOffset() + newBlock.headerSize() + DataBlockEncoding.ID_SIZE, newBlock.getUncompressedSizeWithoutHeader() - DataBlockEncoding.ID_SIZE).slice(); return encodedBlock; }
private ByteBuffer getEncodedBuffer(HFileBlock newBlock) { ByteBuffer origBlock = newBlock.getBufferReadOnly(); ByteBuffer encodedBlock = ByteBuffer.wrap(origBlock.array(), origBlock.arrayOffset() + newBlock.headerSize() + DataBlockEncoding.ID_SIZE, newBlock.getUncompressedSizeWithoutHeader() - DataBlockEncoding.ID_SIZE).slice(); return encodedBlock; }
/** * Called after reading a block with provided onDiskSizeWithHeader. */ private void validateOnDiskSizeWithoutHeader(int expectedOnDiskSizeWithoutHeader) throws IOException { if (onDiskSizeWithoutHeader != expectedOnDiskSizeWithoutHeader) { String dataBegin = null; if (buf.hasArray()) { dataBegin = Bytes.toStringBinary(buf.array(), buf.arrayOffset(), Math.min(32, buf.limit())); } else { ByteBuffer bufDup = getBufferReadOnly(); byte[] dataBeginBytes = new byte[Math.min(32, bufDup.limit() - bufDup.position())]; bufDup.get(dataBeginBytes); dataBegin = Bytes.toStringBinary(dataBeginBytes); } String blockInfoMsg = "Block offset: " + offset + ", data starts with: " + dataBegin; throw new IOException("On-disk size without header provided is " + expectedOnDiskSizeWithoutHeader + ", but block " + "header contains " + onDiskSizeWithoutHeader + ". " + blockInfoMsg); } }
", total data size " + data.length + " Checksum data range offset " + off + " len " + count + HFileBlock.toStringHeader(block.getBufferReadOnly()); HFile.LOG.warn(msg); if (generateExceptions) {
ByteBuffer bloomBuf = bloomBlock.getBufferReadOnly(); result = ByteBloomFilter.contains(key, keyOffset, keyLength, bloomBuf.array(), bloomBuf.arrayOffset() + bloomBlock.headerSize(),
ByteBuffer bloomBuf = bloomBlock.getBufferReadOnly(); result = ByteBloomFilter.contains(key, keyOffset, keyLength, bloomBuf, bloomBlock.headerSize(),
private void testEncodingWithCacheInternals(boolean useTag) throws IOException { List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag); HFileBlock block = getSampleHFileBlock(kvs, useTag); HFileBlock cacheBlock = createBlockOnDisk(kvs, block, useTag); LruBlockCache blockCache = new LruBlockCache(8 * 1024 * 1024, 32 * 1024); BlockCacheKey cacheKey = new BlockCacheKey("test", 0); blockCache.cacheBlock(cacheKey, cacheBlock); HeapSize heapSize = blockCache.getBlock(cacheKey, false, false, true); assertTrue(heapSize instanceof HFileBlock); HFileBlock returnedBlock = (HFileBlock) heapSize;; if (blockEncoder.getDataBlockEncoding() == DataBlockEncoding.NONE) { assertEquals(block.getBufferReadOnly(), returnedBlock.getBufferReadOnly()); } else { if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) { System.out.println(blockEncoder); } assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType()); } }
ByteBuff bufRead = b.getBufferReadOnly(); ByteBuffer bufExpected = expectedContents.get(i); boolean bytesAreCorrect = Bytes.compareTo(bufRead.array(),
if (block.getBlockType() != BlockType.LEAF_INDEX) return; ByteBuff b = block.getBufferReadOnly(); int n = b.getIntAfterPosition(0);