private void __jamon_innerUnit__bc_by_file(final java.io.Writer jamonWriter, final CachedBlocksByFile cbsbf) throws java.io.IOException { // 69, 1 jamonWriter.write("["); // 69, 2 for (Map.Entry<String, NavigableSet<CachedBlock>> e: cbsbf.getCachedBlockStatsByFile().entrySet() ) { // 69, 103 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(BlockCacheUtil.toJSON(e.getKey(), e.getValue())), jamonWriter); } // 69, 163 jamonWriter.write("]\n"); }
return; CachedBlocksByFile cbsbf = BlockCacheUtil.getLoadedCachedBlocksByFile(conf, bc); org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(BlockCacheUtil.toJSON(bc)), jamonWriter); org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(BlockCacheUtil.toJSON(cbsbf)), jamonWriter);
/** * @param cb * @return The block content as String. */ public static String toString(final CachedBlock cb, final long now) { return "filename=" + cb.getFilename() + ", " + toStringMinusFileName(cb, now); }
String bcName = bc.getClass().getSimpleName(); org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.CachedBlocksByFile cbsbf = org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.getLoadedCachedBlocksByFile(config, bc); AgeSnapshot cbsbfSnapshot = cbsbf.getAgeInCacheSnapshot();
/** * Cache the block to ramCache * @param cacheKey block's cache key * @param cachedItem block buffer * @param inMemory if block is in-memory * @param wait if true, blocking wait when queue is full */ private void cacheBlockWithWait(BlockCacheKey cacheKey, Cacheable cachedItem, boolean inMemory, boolean wait) { if (cacheEnabled) { if (backingMap.containsKey(cacheKey) || ramCache.containsKey(cacheKey)) { if (BlockCacheUtil.shouldReplaceExistingCacheBlock(this, cacheKey, cachedItem)) { cacheBlockWithWaitInternal(cacheKey, cachedItem, inMemory, wait); } } else { cacheBlockWithWaitInternal(cacheKey, cachedItem, inMemory, wait); } } }
/** * Validate that the existing and newBlock are the same without including the nextBlockMetadata, * if not, throw an exception. If they are the same without the nextBlockMetadata, * return the comparison. * * @param existing block that is existing in the cache. * @param newBlock block that is trying to be cached. * @param cacheKey the cache key of the blocks. * @return comparison of the existing block to the newBlock. */ public static int validateBlockAddition(Cacheable existing, Cacheable newBlock, BlockCacheKey cacheKey) { int comparison = compareCacheBlock(existing, newBlock, false); if (comparison != 0) { throw new RuntimeException("Cached block contents differ, which should not have happened." + "cacheKey:" + cacheKey); } if ((existing instanceof HFileBlock) && (newBlock instanceof HFileBlock)) { comparison = ((HFileBlock) existing).getNextBlockOnDiskSize() - ((HFileBlock) newBlock).getNextBlockOnDiskSize(); } return comparison; }
Cacheable existingBlock = blockCache.getBlock(cacheKey, false, false, false); try { int comparison = BlockCacheUtil.validateBlockAddition(existingBlock, newBlock, cacheKey); if (comparison < 0) { LOG.warn("Cached block contents differ by nextBlockOnDiskSize, the new block has "
String bcName = bc.getClass().getSimpleName(); org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.CachedBlocksByFile cbsbf = org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.getLoadedCachedBlocksByFile(config, bc); AgeSnapshot cbsbfSnapshot = cbsbf.getAgeInCacheSnapshot();
if (cb != null && !BlockCacheUtil.shouldReplaceExistingCacheBlock(this, cacheKey, buf)) { return;
private BlockCacheUtil.CachedBlocksByFile logPerBlock(final BlockCache bc) throws IOException { BlockCacheUtil.CachedBlocksByFile cbsbf = new BlockCacheUtil.CachedBlocksByFile(); for (CachedBlock cb : bc) { LOG.info(cb.toString()); LOG.info(BlockCacheUtil.toJSON(bc)); cbsbf.update(cb); } return cbsbf; } }
return; CachedBlocksByFile cbsbf = BlockCacheUtil.getLoadedCachedBlocksByFile(conf, bc); org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(BlockCacheUtil.toJSON(bc)), jamonWriter); org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(BlockCacheUtil.toJSON(cbsbf)), jamonWriter);
/** * @param cb * @return The block content as String. */ public static String toString(final CachedBlock cb, final long now) { return "filename=" + cb.getFilename() + ", " + toStringMinusFileName(cb, now); }
private void logPerFile(final BlockCacheUtil.CachedBlocksByFile cbsbf) throws IOException { for (Map.Entry<String, NavigableSet<CachedBlock>> e: cbsbf.getCachedBlockStatsByFile().entrySet()) { int count = 0; long size = 0; int countData = 0; long sizeData = 0; for (CachedBlock cb: e.getValue()) { count++; size += cb.getSize(); BlockType bt = cb.getBlockType(); if (bt != null && bt.isData()) { countData++; sizeData += cb.getSize(); } } LOG.info("filename=" + e.getKey() + ", count=" + count + ", countData=" + countData + ", size=" + size + ", sizeData=" + sizeData); LOG.info(BlockCacheUtil.toJSON(e.getKey(), e.getValue())); } }
@Test public void testBucketCache() throws IOException { this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 100); BlockCache blockCache = BlockCacheFactory.createBlockCache(this.conf); assertTrue(blockCache instanceof CombinedBlockCache); logPerBlock(blockCache); final int count = 3; addDataAndHits(blockCache, count); // The below has no asserts. It is just exercising toString and toJSON code. LOG.info(Objects.toString(blockCache.getStats())); BlockCacheUtil.CachedBlocksByFile cbsbf = logPerBlock(blockCache); LOG.info(Objects.toString(cbsbf)); logPerFile(cbsbf); bucketCacheReport(blockCache); LOG.info(BlockCacheUtil.toJSON(cbsbf)); }
@Test public void testLruBlockCache() throws IOException { CacheConfig cc = new CacheConfig(this.conf); assertTrue(CacheConfig.DEFAULT_IN_MEMORY == cc.isInMemory()); BlockCache blockCache = BlockCacheFactory.createBlockCache(this.conf); assertTrue(blockCache instanceof LruBlockCache); logPerBlock(blockCache); addDataAndHits(blockCache, 3); // The below has no asserts. It is just exercising toString and toJSON code. LOG.info("count=" + blockCache.getBlockCount() + ", currentSize=" + blockCache.getCurrentSize() + ", freeSize=" + blockCache.getFreeSize()); LOG.info(Objects.toString(blockCache.getStats())); BlockCacheUtil.CachedBlocksByFile cbsbf = logPerBlock(blockCache); LOG.info(Objects.toString(cbsbf)); logPerFile(cbsbf); bucketCacheReport(blockCache); LOG.info(BlockCacheUtil.toJSON(cbsbf)); }
private void __jamon_innerUnit__bc_by_file(final java.io.Writer jamonWriter, final CachedBlocksByFile cbsbf) throws java.io.IOException { // 69, 1 jamonWriter.write("["); // 69, 2 for (Map.Entry<String, NavigableSet<CachedBlock>> e: cbsbf.getCachedBlockStatsByFile().entrySet() ) { // 69, 103 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(BlockCacheUtil.toJSON(e.getKey(), e.getValue())), jamonWriter); } // 69, 163 jamonWriter.write("]\n"); }
private BlockCacheUtil.CachedBlocksByFile logPerBlock(final BlockCache bc) throws IOException { BlockCacheUtil.CachedBlocksByFile cbsbf = new BlockCacheUtil.CachedBlocksByFile(); for (CachedBlock cb : bc) { LOG.info(cb.toString()); LOG.info(BlockCacheUtil.toJSON(bc)); cbsbf.update(cb); } return cbsbf; } }
private void logPerFile(final BlockCacheUtil.CachedBlocksByFile cbsbf) throws IOException { for (Map.Entry<String, NavigableSet<CachedBlock>> e: cbsbf.getCachedBlockStatsByFile().entrySet()) { int count = 0; long size = 0; int countData = 0; long sizeData = 0; for (CachedBlock cb: e.getValue()) { count++; size += cb.getSize(); BlockType bt = cb.getBlockType(); if (bt != null && bt.isData()) { countData++; sizeData += cb.getSize(); } } LOG.info("filename=" + e.getKey() + ", count=" + count + ", countData=" + countData + ", size=" + size + ", sizeData=" + sizeData); LOG.info(BlockCacheUtil.toJSON(e.getKey(), e.getValue())); } }
@Test public void testBucketCache() throws IOException { this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 100); CacheConfig.instantiateBlockCache(this.conf); CacheConfig cc = new CacheConfig(this.conf); assertTrue(cc.getBlockCache() instanceof CombinedBlockCache); logPerBlock(cc.getBlockCache()); final int count = 3; addDataAndHits(cc.getBlockCache(), count); // The below has no asserts. It is just exercising toString and toJSON code. LOG.info(Objects.toString(cc.getBlockCache().getStats())); BlockCacheUtil.CachedBlocksByFile cbsbf = logPerBlock(cc.getBlockCache()); LOG.info(Objects.toString(cbsbf)); logPerFile(cbsbf); bucketCacheReport(cc.getBlockCache()); LOG.info(BlockCacheUtil.toJSON(cbsbf)); }
@Test public void testLruBlockCache() throws IOException { CacheConfig.instantiateBlockCache(this.conf); CacheConfig cc = new CacheConfig(this.conf); assertTrue(cc.isBlockCacheEnabled()); assertTrue(CacheConfig.DEFAULT_IN_MEMORY == cc.isInMemory()); assertTrue(cc.getBlockCache() instanceof LruBlockCache); logPerBlock(cc.getBlockCache()); addDataAndHits(cc.getBlockCache(), 3); // The below has no asserts. It is just exercising toString and toJSON code. BlockCache bc = cc.getBlockCache(); LOG.info("count=" + bc.getBlockCount() + ", currentSize=" + bc.getCurrentSize() + ", freeSize=" + bc.getFreeSize() ); LOG.info(Objects.toString(cc.getBlockCache().getStats())); BlockCacheUtil.CachedBlocksByFile cbsbf = logPerBlock(cc.getBlockCache()); LOG.info(Objects.toString(cbsbf)); logPerFile(cbsbf); bucketCacheReport(cc.getBlockCache()); LOG.info(BlockCacheUtil.toJSON(cbsbf)); }