/** * * @param cacheKey The block's cache key. * @param buf The block contents wrapped in a ByteBuffer. * @param inMemory Whether block should be treated as in-memory. This parameter is only useful for * the L1 lru cache. */ @Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { // This is the inclusive part of the combined block cache. // Every block is placed into both block caches. onHeapCache.cacheBlock(cacheKey, buf, inMemory); // This assumes that insertion into the L2 block cache is either async or very fast. l2Cache.cacheBlock(cacheKey, buf, inMemory); }
@Override public void doAnAction() throws Exception { toBeTested.evictBlock(key); toBeTested.cacheBlock(key, bac); Thread.sleep(blockEvictPeriod); } };
/** * Caches the last written HFile block. * @param offset the offset of the block we want to cache. Used to determine * the cache key. */ private void doCacheOnWrite(long offset) { cacheConf.getBlockCache().ifPresent(cache -> { HFileBlock cacheFormatBlock = blockWriter.getBlockForCaching(cacheConf); cache.cacheBlock(new BlockCacheKey(name, offset, true, cacheFormatBlock.getBlockType()), cacheFormatBlock); }); }
@Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { boolean metaBlock = buf.getBlockType().getCategory() != BlockCategory.DATA; if (metaBlock) { onHeapCache.cacheBlock(cacheKey, buf, inMemory); } else { l2Cache.cacheBlock(cacheKey, buf, inMemory); } }
@Override public void doAnAction() throws Exception { for (int j = 0; j < 100; j++) { BlockCacheKey key = new BlockCacheKey("key_" + finalI + "_" + j, 0); Arrays.fill(buf, (byte) (finalI * j)); final ByteArrayCacheable bac = new ByteArrayCacheable(buf); ByteArrayCacheable gotBack = (ByteArrayCacheable) toBeTested .getBlock(key, true, false, true); if (gotBack != null) { assertArrayEquals(gotBack.buf, bac.buf); } else { toBeTested.cacheBlock(key, bac); } } totalQueries.incrementAndGet(); } };
public static void getBlockAndAssertEquals(BlockCache cache, BlockCacheKey key, Cacheable blockToCache, ByteBuffer destBuffer, ByteBuffer expectedBuffer) { destBuffer.clear(); cache.cacheBlock(key, blockToCache); Cacheable actualBlock = cache.getBlock(key, false, false, false); actualBlock.serialize(destBuffer, true); assertEquals(expectedBuffer, destBuffer); cache.returnBlock(key, actualBlock); } }
@Override public void doAnAction() throws Exception { if (!blocksToTest.isEmpty()) { HFileBlockPair ourBlock = blocksToTest.poll(); // if we run out of blocks to test, then we should stop the tests. if (ourBlock == null) { ctx.setStopFlag(true); return; } toBeTested.cacheBlock(ourBlock.blockName, ourBlock.block); Cacheable retrievedBlock = toBeTested.getBlock(ourBlock.blockName, false, false, true); if (retrievedBlock != null) { assertEquals(ourBlock.block, retrievedBlock); toBeTested.evictBlock(ourBlock.blockName); hits.incrementAndGet(); assertNull(toBeTested.getBlock(ourBlock.blockName, false, false, true)); } else { miss.incrementAndGet(); } totalQueries.incrementAndGet(); } } };
victimHandler.cacheBlock(block.getCacheKey(), block.getBuffer());
.ifPresent(cache -> cache.cacheBlock(cacheKey, metaBlock, cacheConf.isInMemory()));
cacheConf.getBlockCache().ifPresent(cache -> { HFileBlock blockForCaching = blockWriter.getBlockForCaching(cacheConf); cache.cacheBlock(new BlockCacheKey(nameForCaching, rootLevelIndexPos, true, blockForCaching.getBlockType()), blockForCaching); });
toBeTested.cacheBlock(block.blockName, block.block); try { if (toBeTested.getBlock(block.blockName, true, false, true) != null) { toBeTested.cacheBlock(block.blockName, block.block); if (!(toBeTested instanceof BucketCache)) {
toBeTested.cacheBlock(key, bac);
cache.cacheBlock(cacheKey, cacheConf.shouldCacheCompressed(category) ? hfileBlock : unpacked, cacheConf.isInMemory());
cacheConf.getBlockCache().ifPresent(cache -> { HFileBlock blockForCaching = blockWriter.getBlockForCaching(cacheConf); cache.cacheBlock( new BlockCacheKey(nameForCaching, beginOffset, true, blockForCaching.getBlockType()), blockForCaching);
/** * Just checks if heapsize grows when something is cached, and gets smaller * when the same object is evicted */ public static void testHeapSizeChanges(final BlockCache toBeTested, final int blockSize) { HFileBlockPair[] blocks = generateHFileBlocks(blockSize, 1); long heapSize = ((HeapSize) toBeTested).heapSize(); toBeTested.cacheBlock(blocks[0].blockName, blocks[0].block); /*When we cache something HeapSize should always increase */ assertTrue(heapSize < ((HeapSize) toBeTested).heapSize()); toBeTested.evictBlock(blocks[0].blockName); /*Post eviction, heapsize should be the same */ assertEquals(heapSize, ((HeapSize) toBeTested).heapSize()); }
private void addDataAndHits(final BlockCache bc, final int count) { Cacheable dce = new DataCacheEntry(); Cacheable ice = new IndexCacheEntry(); for (int i = 0; i < count; i++) { BlockCacheKey bckd = new BlockCacheKey("f", i); BlockCacheKey bcki = new BlockCacheKey("f", i + count); bc.getBlock(bckd, true, false, true); bc.cacheBlock(bckd, dce); bc.cacheBlock(bcki, ice); bc.getBlock(bckd, true, false, true); bc.getBlock(bcki, true, false, true); } assertEquals(2 * count /*Data and Index blocks*/, bc.getStats().getHitCount()); BlockCacheKey bckd = new BlockCacheKey("f", 0); BlockCacheKey bcki = new BlockCacheKey("f", 0 + count); bc.evictBlock(bckd); bc.evictBlock(bcki); bc.getStats().getEvictedCount(); }
/** * @param bc The block cache instance. * @param cc Cache config. * @param doubling If true, addition of element ups counter by 2, not 1, because element added * to onheap and offheap caches. * @param sizing True if we should run sizing test (doesn't always apply). */ void basicBlockCacheOps(final BlockCache bc, final CacheConfig cc, final boolean doubling, final boolean sizing) { assertTrue(CacheConfig.DEFAULT_IN_MEMORY == cc.isInMemory()); BlockCacheKey bck = new BlockCacheKey("f", 0); Cacheable c = new DataCacheEntry(); // Do asserts on block counting. long initialBlockCount = bc.getBlockCount(); bc.cacheBlock(bck, c, cc.isInMemory()); assertEquals(doubling ? 2 : 1, bc.getBlockCount() - initialBlockCount); bc.evictBlock(bck); assertEquals(initialBlockCount, bc.getBlockCount()); // Do size accounting. Do it after the above 'warm-up' because it looks like some // buffers do lazy allocation so sizes are off on first go around. if (sizing) { long originalSize = bc.getCurrentSize(); bc.cacheBlock(bck, c, cc.isInMemory()); assertTrue(bc.getCurrentSize() > originalSize); bc.evictBlock(bck); long size = bc.getCurrentSize(); assertEquals(originalSize, size); } }
@Override public void doAnAction() throws Exception { toBeTested.evictBlock(key); toBeTested.cacheBlock(key, bac); Thread.sleep(blockEvictPeriod); } };
@Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory, final boolean cacheDataInL1) { boolean isMetaBlock = buf.getBlockType().getCategory() != BlockCategory.DATA; if (isMetaBlock || cacheDataInL1) { lruCache.cacheBlock(cacheKey, buf, inMemory, cacheDataInL1); } else { l2Cache.cacheBlock(cacheKey, buf, inMemory, false); } }
public static void getBlockAndAssertEquals(BlockCache cache, BlockCacheKey key, Cacheable blockToCache, ByteBuffer destBuffer, ByteBuffer expectedBuffer) { destBuffer.clear(); cache.cacheBlock(key, blockToCache); Cacheable actualBlock = cache.getBlock(key, false, false, false); actualBlock.serialize(destBuffer, true); assertEquals(expectedBuffer, destBuffer); cache.returnBlock(key, actualBlock); } }