@Test public void testDisableCacheDataBlock() throws IOException { Configuration conf = HBaseConfiguration.create(); CacheConfig cacheConfig = new CacheConfig(conf); assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA)); assertFalse(cacheConfig.shouldCacheCompressed(BlockCategory.DATA)); assertFalse(cacheConfig.shouldCacheDataCompressed()); assertFalse(cacheConfig.shouldCacheDataOnWrite()); assertTrue(cacheConfig.shouldCacheDataOnRead()); assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.INDEX)); assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.META)); assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.BLOOM)); assertFalse(cacheConfig.shouldCacheBloomsOnWrite()); assertFalse(cacheConfig.shouldCacheIndexesOnWrite()); cacheConfig = new CacheConfig(conf); assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA)); assertTrue(cacheConfig.shouldCacheCompressed(BlockCategory.DATA)); assertTrue(cacheConfig.shouldCacheDataCompressed()); assertTrue(cacheConfig.shouldCacheDataOnWrite()); assertTrue(cacheConfig.shouldCacheDataOnRead()); assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.INDEX)); assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.META)); assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.BLOOM)); assertTrue(cacheConfig.shouldCacheBloomsOnWrite()); assertTrue(cacheConfig.shouldCacheIndexesOnWrite()); cacheConfig = new CacheConfig(conf); assertFalse(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA)); assertFalse(cacheConfig.shouldCacheCompressed(BlockCategory.DATA));
private BlockCache setCacheProperties(HRegion region) { Iterator<HStore> strItr = region.getStores().iterator(); BlockCache cache = null; while (strItr.hasNext()) { HStore store = strItr.next(); CacheConfig cacheConf = store.getCacheConfig(); cacheConf.setCacheDataOnWrite(true); cacheConf.setEvictOnClose(true); // Use the last one cache = cacheConf.getBlockCache().get(); } return cache; }
while (true) { if (cacheConf.shouldReadBlockFromCache(expectedBlockType)) { if (useLock) { lockEntry = offsetLock.getLockEntry(dataBlockOffset); if (!useLock && cacheBlock && cacheConf.shouldLockOnCacheMiss(expectedBlockType)) { cacheConf.getBlockCache().ifPresent(cache -> { if (cacheBlock && cacheConf.shouldCacheBlockOnRead(category)) { cache.cacheBlock(cacheKey, cacheConf.shouldCacheCompressed(category) ? hfileBlock : unpacked, cacheConf.isInMemory());
@Override public String toString() { return "cacheDataOnRead=" + shouldCacheDataOnRead() + ", cacheDataOnWrite=" + shouldCacheDataOnWrite() + ", cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + ", cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + ", cacheEvictOnClose=" + shouldEvictOnClose() + ", cacheDataCompressed=" + shouldCacheDataCompressed() + ", prefetchOnOpen=" + shouldPrefetchOnOpen(); } }
@Test public void testPrefetchSetInHCDWorks() { ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f")).setPrefetchBlocksOnOpen(true) .build(); Configuration c = HBaseConfiguration.create(); assertFalse(c.getBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, false)); CacheConfig cc = new CacheConfig(c, columnFamilyDescriptor, blockCache); assertTrue(cc.shouldPrefetchOnOpen()); }
/** * Creates the cache config. * @param family The current column family. */ protected void createCacheConf(final ColumnFamilyDescriptor family) { this.cacheConf = new CacheConfig(conf, family, region.getBlockCache()); }
if (isCompaction) { writerCacheConf = new CacheConfig(cacheConf); writerCacheConf.setCacheDataOnWrite(false); } else { writerCacheConf = cacheConf;
@Test public void testCacheConfigDefaultLRUBlockCache() { CacheConfig.instantiateBlockCache(this.conf); CacheConfig cc = new CacheConfig(this.conf); assertTrue(cc.isBlockCacheEnabled()); assertTrue(CacheConfig.DEFAULT_IN_MEMORY == cc.isInMemory()); basicBlockCacheOps(cc, false, true); assertTrue(cc.getBlockCache() instanceof LruBlockCache); }
this.isPrimaryReplicaReader()); cacheBlock &= cacheConf.shouldCacheDataOnRead(); if (cacheConf.isBlockCacheEnabled()) { HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, true, BlockType.META, null); cacheConf.getBlockCache().cacheBlock(cacheKey, metaBlock, cacheConf.isInMemory(), this.cacheConf.isCacheDataInL1());
lazyCompressDisabled.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, cacheOnWrite); lazyCompressDisabled.setBoolean(CacheConfig.CACHE_DATA_BLOCKS_COMPRESSED_KEY, false); CacheConfig cc = new CacheConfig(lazyCompressDisabled, new LruBlockCache(maxSize, HConstants.DEFAULT_BLOCKSIZE, false, lazyCompressDisabled)); assertFalse(cc.shouldCacheDataCompressed()); assertTrue(cc.getBlockCache().get() instanceof LruBlockCache); LruBlockCache disabledBlockCache = (LruBlockCache) cc.getBlockCache().get(); LOG.info("disabledBlockCache=" + disabledBlockCache); assertEquals("test inconsistency detected.", maxSize, disabledBlockCache.getMaxSize()); lazyCompressEnabled.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, cacheOnWrite); lazyCompressEnabled.setBoolean(CacheConfig.CACHE_DATA_BLOCKS_COMPRESSED_KEY, true); cc = new CacheConfig(lazyCompressEnabled, new LruBlockCache(maxSize, HConstants.DEFAULT_BLOCKSIZE, false, lazyCompressEnabled)); assertTrue("test improperly configured.", cc.shouldCacheDataCompressed()); assertTrue(cc.getBlockCache().get() instanceof LruBlockCache); LruBlockCache enabledBlockCache = (LruBlockCache) cc.getBlockCache().get(); LOG.info("enabledBlockCache=" + enabledBlockCache); assertEquals("test inconsistency detected", maxSize, enabledBlockCache.getMaxSize()); candidatesFound++; HFileBlock block = (HFileBlock) e.getValue().getBuffer(); if (cc.shouldCacheCompressed(block.getBlockType().getCategory())) { assertFalse("found an unpacked block, block=" + block + ", block buffer capacity=" + block.getBufferWithoutHeader().capacity(), block.isUnpacked());
@Override public String toString() { if (!isBlockCacheEnabled()) { return "CacheConfig:disabled"; } return "blockCache=" + getBlockCache() + ", cacheDataOnRead=" + shouldCacheDataOnRead() + ", cacheDataOnWrite=" + shouldCacheDataOnWrite() + ", cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + ", cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + ", cacheEvictOnClose=" + shouldEvictOnClose() + ", cacheDataCompressed=" + shouldCacheDataCompressed() + ", prefetchOnOpen=" + shouldPrefetchOnOpen(); }
@Override public String toString() { if (!isBlockCacheEnabled()) { return "CacheConfig:disabled"; } return "CacheConfig:enabled " + "[cacheDataOnRead=" + shouldCacheDataOnRead() + "] " + "[cacheDataOnWrite=" + shouldCacheDataOnWrite() + "] " + "[cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + "] " + "[cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + "] " + "[cacheEvictOnClose=" + shouldEvictOnClose() + "] " + "[cacheCompressed=" + shouldCacheCompressed() + "]"; }
protected CacheConfig getCacheConfInstance(BlockCache blockCache, SortedOplogStatistics stats, HFileStoreStatistics storeStats) { CacheConfig tmpConfig = null; // if (stats == null) { tmpConfig = new CacheConfig(conf); // } else { // tmpConfig = new CacheConfig(conf, CACHE_DATA_BLOCKS_ON_READ, blockCache, // HFileSortedOplogFactory.convertStatistics(stats, storeStats)); // } tmpConfig.shouldCacheBlockOnRead(BlockCategory.ALL_CATEGORIES); return tmpConfig; }
@Test public void testCacheConfigDefaultLRUBlockCache() { CacheConfig cc = new CacheConfig(this.conf); assertTrue(CacheConfig.DEFAULT_IN_MEMORY == cc.isInMemory()); BlockCache blockCache = BlockCacheFactory.createBlockCache(this.conf); basicBlockCacheOps(blockCache, cc, false, true); assertTrue(blockCache instanceof LruBlockCache); }
CacheConfig cacheConf = new CacheConfig(conf, BlockCacheFactory.createBlockCache(conf)); BlockCache blockCache = cacheConf.getBlockCache().get();
this.isPrimaryReplicaReader(), BlockType.META); cacheBlock &= cacheConf.shouldCacheBlockOnRead(BlockType.META.getCategory()); HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, true, BlockType.META, null); cacheConf.getBlockCache() .ifPresent(cache -> cache.cacheBlock(cacheKey, metaBlock, cacheConf.isInMemory()));
@Override public void close(boolean evictOnClose) throws IOException { PrefetchExecutor.cancel(path); cacheConf.getBlockCache().ifPresent(cache -> { if (evictOnClose) { int numEvicted = cache.evictBlocksByHfileName(name); if (LOG.isTraceEnabled()) { LOG.trace("On close, file=" + name + " evicted=" + numEvicted + " block(s)"); } } }); fsBlockReader.closeStreams(); }
DataBlockEncoding expectedDataBlockEncoding) throws IOException { if (cacheConf.isBlockCacheEnabled()) { BlockCache cache = cacheConf.getBlockCache(); HFileBlock cachedBlock = (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock, updateCacheMetrics); if (cachedBlock != null) { if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) { cachedBlock = cachedBlock.unpack(hfileContext, fsBlockReader);
/** * @param cc * @param doubling If true, addition of element ups counter by 2, not 1, because element added * to onheap and offheap caches. * @param sizing True if we should run sizing test (doesn't always apply). */ void basicBlockCacheOps(final CacheConfig cc, final boolean doubling, final boolean sizing) { assertTrue(cc.isBlockCacheEnabled()); assertTrue(CacheConfig.DEFAULT_IN_MEMORY == cc.isInMemory()); BlockCache bc = cc.getBlockCache(); BlockCacheKey bck = new BlockCacheKey("f", 0); Cacheable c = new DataCacheEntry(); // Do asserts on block counting. long initialBlockCount = bc.getBlockCount(); bc.cacheBlock(bck, c, cc.isInMemory()); assertEquals(doubling? 2: 1, bc.getBlockCount() - initialBlockCount); bc.evictBlock(bck); assertEquals(initialBlockCount, bc.getBlockCount()); // Do size accounting. Do it after the above 'warm-up' because it looks like some // buffers do lazy allocation so sizes are off on first go around. if (sizing) { long originalSize = bc.getCurrentSize(); bc.cacheBlock(bck, c, cc.isInMemory()); assertTrue(bc.getCurrentSize() > originalSize); bc.evictBlock(bck); long size = bc.getCurrentSize(); assertEquals(originalSize, size); } }
DataBlockEncoding expectedDataBlockEncoding) throws IOException { BlockCache cache = cacheConf.getBlockCache().orElse(null); if (cache != null) { HFileBlock cachedBlock = (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock, updateCacheMetrics); if (cachedBlock != null) { if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) { HFileBlock compressedBlock = cachedBlock; cachedBlock = compressedBlock.unpack(hfileContext, fsBlockReader);