/** * Cache the block with the specified name and buffer. * <p> * * @param cacheKey block's cache key * @param buf block buffer */ @Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { cacheBlock(cacheKey, buf, false); }
/** * * @param cacheKey The block's cache key. * @param buf The block contents wrapped in a ByteBuffer. * @param inMemory Whether block should be treated as in-memory. This parameter is only useful for * the L1 lru cache. */ @Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { // This is the inclusive part of the combined block cache. // Every block is placed into both block caches. onHeapCache.cacheBlock(cacheKey, buf, inMemory); // This assumes that insertion into the L2 block cache is either async or very fast. l2Cache.cacheBlock(cacheKey, buf, inMemory); }
@Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { boolean metaBlock = buf.getBlockType().getCategory() != BlockCategory.DATA; if (metaBlock) { onHeapCache.cacheBlock(cacheKey, buf, inMemory); } else { l2Cache.cacheBlock(cacheKey, buf, inMemory); } }
@Override public void run() { for (int blockIndex = 0; blockIndex < blocksPerThread || (!cache.isEvictionInProgress()); ++blockIndex) { CachedItem block = new CachedItem(hfileName, (int) blockSize, blockCount.getAndIncrement()); boolean inMemory = Math.random() > 0.5; cache.cacheBlock(block.cacheKey, block, inMemory); } cache.evictBlocksByHfileName(hfileName); } });
result = ((HFileBlock) result).deepClone(); cacheBlock(cacheKey, result, /* inMemory = */ false);
cache.cacheBlock(i.cacheKey, i); cache.cacheBlock(i.cacheKey, i);
cache.cacheBlock(block.cacheKey, block);
cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]); expectedCacheSize += singleBlocks[i].cacheBlockHeapSize(); cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]); expectedCacheSize += multiBlocks[i].cacheBlockHeapSize(); cache.getBlock(multiBlocks[i].cacheKey, true, false, true); cache.cacheBlock(singleBlocks[4].cacheKey, singleBlocks[4]); expectedCacheSize += singleBlocks[4].cacheBlockHeapSize(); cache.cacheBlock(memoryBlocks[0].cacheKey, memoryBlocks[0], true); cache.cacheBlock(memoryBlocks[1].cacheKey, memoryBlocks[1], true); cache.cacheBlock(memoryBlocks[2].cacheKey, memoryBlocks[2], true); cache.cacheBlock(memoryBlocks[3].cacheKey, memoryBlocks[3], true); cache.cacheBlock(memoryBlocks[4].cacheKey, memoryBlocks[4], true); cache.cacheBlock(memoryBlocks[5].cacheKey, memoryBlocks[5], true); cache.cacheBlock(memoryBlocks[6].cacheKey, memoryBlocks[6], true); cache.cacheBlock(memoryBlocks[7].cacheKey, memoryBlocks[7], true); cache.cacheBlock(memoryBlocks[8].cacheKey, memoryBlocks[8], true); cache.cacheBlock(memoryBlocks[9].cacheKey, memoryBlocks[9], true); cache.cacheBlock(singleBlocks[9].cacheKey, singleBlocks[9]);
private void testEncodingWithCacheInternals(boolean useTag) throws IOException { List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag); HFileBlock block = getSampleHFileBlock(kvs, useTag); HFileBlock cacheBlock = createBlockOnDisk(kvs, block, useTag); LruBlockCache blockCache = new LruBlockCache(8 * 1024 * 1024, 32 * 1024); BlockCacheKey cacheKey = new BlockCacheKey("test", 0); blockCache.cacheBlock(cacheKey, cacheBlock); HeapSize heapSize = blockCache.getBlock(cacheKey, false, false, true); assertTrue(heapSize instanceof HFileBlock); HFileBlock returnedBlock = (HFileBlock) heapSize; if (blockEncoder.getDataBlockEncoding() == DataBlockEncoding.NONE) { assertEquals(block.getBufferReadOnly(), returnedBlock.getBufferReadOnly()); } else { if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) { System.out.println(blockEncoder); } assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType()); } }
Cacheable c = new DataCacheEntry(); BlockCacheKey bck = new BlockCacheKey("bck", 0); lbc.cacheBlock(bck, c, false); assertEquals(initialL1BlockCount + 1, lbc.getBlockCount()); assertEquals(initialL2BlockCount, bc.getBlockCount()); lbc.cacheBlock(new BlockCacheKey("bck2", 0), new DataCacheEntry() { @Override public long heapSize() {
cache.cacheBlock(block.cacheKey, block); cache.getBlock(block.cacheKey, true, false, true); cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]); cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]); cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]); cache.getBlock(multiBlocks[i].cacheKey, true, false, true); cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true);
cache.cacheBlock(block.cacheKey, block); expectedCacheSize += block.cacheBlockHeapSize(); cache.cacheBlock(block.cacheKey, block);
cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]); expectedCacheSize += singleBlocks[i].cacheBlockHeapSize(); cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]); expectedCacheSize += multiBlocks[i].cacheBlockHeapSize(); cache.getBlock(multiBlocks[i].cacheKey, true, false, true); cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true); expectedCacheSize += memoryBlocks[i].cacheBlockHeapSize(); cache.cacheBlock(singleBlocks[3].cacheKey, singleBlocks[3]); cache.cacheBlock(singleBlocks[4].cacheKey, singleBlocks[4]); cache.cacheBlock(memoryBlocks[3].cacheKey, memoryBlocks[3], true); cache.cacheBlock(bigBlocks[0].cacheKey, bigBlocks[0]); cache.cacheBlock(bigBlocks[1].cacheKey, bigBlocks[1]); cache.cacheBlock(bigBlocks[2].cacheKey, bigBlocks[2], true);
cache.cacheBlock(block.cacheKey, block); expectedCacheSize += block.cacheBlockHeapSize(); assertEquals(cache.getBlock(block.cacheKey, true, false, true), block); cache.cacheBlock(block.cacheKey, block); expectedCacheSize += block.heapSize();
@Test public void testCacheEvictionSimple() throws Exception { long maxSize = 100000; long blockSize = calculateBlockSizeDefault(maxSize, 10); LruBlockCache cache = new LruBlockCache(maxSize,blockSize,false); CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block"); long expectedCacheSize = cache.heapSize(); // Add all the blocks for (CachedItem block : blocks) { cache.cacheBlock(block.cacheKey, block); expectedCacheSize += block.cacheBlockHeapSize(); } // A single eviction run should have occurred assertEquals(1, cache.getStats().getEvictionCount()); // Our expected size overruns acceptable limit assertTrue(expectedCacheSize > (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR)); // But the cache did not grow beyond max assertTrue(cache.heapSize() < maxSize); // And is still below the acceptable limit assertTrue(cache.heapSize() < (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR)); // All blocks except block 0 should be in the cache assertTrue(cache.getBlock(blocks[0].cacheKey, true, false, true) == null); for(int i=1;i<blocks.length;i++) { assertEquals(cache.getBlock(blocks[i].cacheKey, true, false, true), blocks[i]); } }
/** * Cache the block with the specified name and buffer. * <p> * @param cacheKey block's cache key * @param buf block buffer */ public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { cacheBlock(cacheKey, buf, false, false); }
@Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { onHeapCache.cacheBlock(cacheKey, buf, inMemory); offHeapCache.cacheBlock(cacheKey, buf); }
@Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { onHeapCache.cacheBlock(cacheKey, buf); offHeapCache.cacheBlock(cacheKey, buf); }
@Override public void run() { for (int blockIndex = 0; blockIndex < blocksPerThread || (!cache.isEvictionInProgress()); ++blockIndex) { CachedItem block = new CachedItem(hfileName, (int) blockSize, blockCount.getAndIncrement()); boolean inMemory = Math.random() > 0.5; cache.cacheBlock(block.cacheKey, block, inMemory); } cache.evictBlocksByHfileName(hfileName); } });