@Override public boolean evictBlock(BlockCacheKey cacheKey) { try { cacheStats.evict(); return client.delete(cacheKey.toString()).get(); } catch (InterruptedException e) { LOG.warn("Error deleting " + cacheKey.toString(), e); Thread.currentThread().interrupt(); } catch (ExecutionException e) { if (LOG.isDebugEnabled()) { LOG.debug("Error deleting " + cacheKey.toString(), e); } } return false; }
@Override public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, boolean updateCacheMetrics) { // Assume that nothing is the block cache HFileBlock result = null; try (TraceScope traceScope = TraceUtil.createTrace("MemcachedBlockCache.getBlock")) { result = client.get(cacheKey.toString(), tc); } catch (Exception e) { // Catch a pretty broad set of exceptions to limit any changes in the memecache client // and how it handles failures from leaking into the read path. if (LOG.isDebugEnabled()) { LOG.debug("Exception pulling from memcached [ " + cacheKey.toString() + " ]. Treating as a miss.", e); } result = null; } finally { // Update stats if this request doesn't have it turned off 100% of the time if (updateCacheMetrics) { if (result == null) { cacheStats.miss(caching, cacheKey.isPrimary(), cacheKey.getBlockType()); } else { cacheStats.hit(caching, cacheKey.isPrimary(), cacheKey.getBlockType()); } } } return result; }
CacheStats stats = new CacheStats("test", 3); stats.rollMetricsPeriod(); assertEquals(0.0, stats.getHitRatioPastNPeriods(), delta); assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta); stats.hit(false, true, BlockType.DATA); stats.hit(true, true, BlockType.DATA); stats.miss(false, false, BlockType.DATA); stats.miss(false, false, BlockType.DATA); stats.rollMetricsPeriod(); assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta); assertEquals(1.0, stats.getHitCachingRatioPastNPeriods(), delta); stats.miss(true, false, BlockType.DATA); stats.miss(false, false, BlockType.DATA); stats.miss(false, false, BlockType.DATA); stats.miss(false, false, BlockType.DATA); stats.rollMetricsPeriod(); assertEquals(0.25, stats.getHitRatioPastNPeriods(), delta); assertEquals(0.5, stats.getHitCachingRatioPastNPeriods(), delta); stats.hit(false, true, BlockType.DATA); stats.hit(true, true, BlockType.DATA); stats.hit(false, true, BlockType.DATA); stats.hit(true, true, BlockType.DATA); stats.rollMetricsPeriod(); assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta); assertEquals(0.75, stats.getHitCachingRatioPastNPeriods(), delta);
public double evictedPerEviction() { double evictionCount = getEvictionCount(); if (evictionCount == 0) { return 0; } return getEvictedCount() / evictionCount; }
public double getMissRatio() { double requestCount = getRequestCount(); if (requestCount == 0) { return 0; } return getMissCount() / requestCount; }
public double getHitRatio() { double requestCount = getRequestCount(); if (requestCount == 0) { return 0; } return getHitCount() / requestCount; }
public void logStats() { // Log size long totalSize = heapSize(); long freeSize = maxSize - totalSize; LruBlockCache.LOG.info("totalSize=" + StringUtils.byteDesc(totalSize) + ", " + "freeSize=" + StringUtils.byteDesc(freeSize) + ", " + "max=" + StringUtils.byteDesc(this.maxSize) + ", " + "blockCount=" + getBlockCount() + ", " + "accesses=" + stats.getRequestCount() + ", " + "hits=" + stats.getHitCount() + ", " + "hitRatio=" + (stats.getHitCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitRatio(), 2)+ ", ")) + ", " + "cachingAccesses=" + stats.getRequestCachingCount() + ", " + "cachingHits=" + stats.getHitCachingCount() + ", " + "cachingHitsRatio=" + (stats.getHitCachingCount() == 0 ? "0,": (StringUtils.formatPercent(stats.getHitCachingRatio(), 2) + ", ")) + "evictions=" + stats.getEvictionCount() + ", " + "evicted=" + stats.getEvictedCount() + ", " + "evictedPerRun=" + stats.evictedPerEviction()); }
@Test public void testCombinedCacheStats() { CacheStats lruCacheStats = new CacheStats("lruCacheStats", 2); CacheStats bucketCacheStats = new CacheStats("bucketCacheStats", 2); CombinedCacheStats stats = new CombinedCacheStats(lruCacheStats, bucketCacheStats); lruCacheStats.hit(true, true, BlockType.DATA); lruCacheStats.miss(true, false, BlockType.DATA); bucketCacheStats.hit(false,true, BlockType.DATA); bucketCacheStats.hit(false,true, BlockType.DATA); bucketCacheStats.miss(false, true, BlockType.DATA); lruCacheStats.evicted(1000, true); lruCacheStats.evicted(1000, false); lruCacheStats.evict(); bucketCacheStats.evict(); assertEquals(2, stats.getEvictionCount()); assertEquals(2, stats.getEvictedCount()); lruCacheStats.failInsert(); assertEquals(1, stats.getFailedInserts()); lruCacheStats.hit(true, true, BlockType.DATA); lruCacheStats.hit(true, true, BlockType.DATA); lruCacheStats.hit(true, true, BlockType.DATA); stats.rollMetricsPeriod(); assertEquals(6, stats.getSumHitCountsPastNPeriods());
@Override public String toString() { AgeSnapshot snapshot = getAgeAtEvictionSnapshot(); return "hitCount=" + getHitCount() + ", hitCachingCount=" + getHitCachingCount() + ", missCount=" + getMissCount() + ", missCachingCount=" + getMissCachingCount() + ", evictionCount=" + getEvictionCount() + ", evictedBlockCount=" + getEvictedCount() + ", primaryMissCount=" + getPrimaryMissCount() + ", primaryHitCount=" + getPrimaryHitCount() + ", evictedAgeMean=" + snapshot.getMean(); }
public long getRequestCount() { return getHitCount() + getMissCount(); }
public void rollMetricsPeriod() { hitCounts[windowIndex] = getHitCount() - lastHitCount; lastHitCount = getHitCount(); hitCachingCounts[windowIndex] = getHitCachingCount() - lastHitCachingCount; lastHitCachingCount = getHitCachingCount(); requestCounts[windowIndex] = getRequestCount() - lastRequestCount; lastRequestCount = getRequestCount(); requestCachingCounts[windowIndex] = getRequestCachingCount() - lastRequestCachingCount; lastRequestCachingCount = getRequestCachingCount(); windowIndex = (windowIndex + 1) % numPeriodsInWindow; }
this.metrics.blockCacheSize.set(blockCache.getCurrentSize()); CacheStats cacheStats = blockCache.getStats(); this.metrics.blockCacheHitCount.set(cacheStats.getHitCount()); this.metrics.blockCacheMissCount.set(cacheStats.getMissCount()); this.metrics.blockCacheEvictedCount.set(blockCache.getEvictedCount()); double ratio = blockCache.getStats().getHitRatio(); int percent = (int) (ratio * 100); this.metrics.blockCacheHitRatio.set(percent); ratio = blockCache.getStats().getHitCachingRatio(); percent = (int) (ratio * 100); this.metrics.blockCacheHitCachingRatio.set(percent); cacheStats.rollMetricsPeriod(); ratio = cacheStats.getHitRatioPastNPeriods(); percent = (int) (ratio * 100); this.metrics.blockCacheHitRatioPastNPeriods.set(percent); ratio = cacheStats.getHitCachingRatioPastNPeriods(); percent = (int) (ratio * 100); this.metrics.blockCacheHitCachingRatioPastNPeriods.set(percent);
private void __jamon_innerUnit__hits_tmpl(final java.io.Writer jamonWriter, final BlockCache bc) throws java.io.IOException { // 196, 5 jamonWriter.write("<tr>\n <td>Hits</td>\n <td>"); // 198, 13 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,d", bc.getStats().getHitCount())), jamonWriter); // 198, 68 jamonWriter.write("</td>\n <td>Number requests that were cache hits</td>\n </tr>\n <tr>\n <td>Hits Caching</td>\n <td>"); // 203, 13 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,d", bc.getStats().getHitCachingCount())), jamonWriter); // 203, 75 jamonWriter.write("</td>\n <td>Cache hit block requests but only requests set to cache block if a miss</td>\n </tr>\n <tr>\n <td>Misses</td>\n <td>"); // 208, 13 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,d", bc.getStats().getMissCount())), jamonWriter); // 208, 69 jamonWriter.write("</td>\n <td>Block requests that were cache misses but set to cache missed blocks</td>\n </tr>\n <tr>\n <td>Misses Caching</td>\n <td>"); // 213, 13 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,d", bc.getStats().getMissCount())), jamonWriter); // 213, 69 jamonWriter.write("</td>\n <td>Block requests that were cache misses but only requests set to use block cache</td>\n </tr>\n <tr>\n <td>Hit Ratio</td>\n <td>"); // 218, 13 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,.2f", bc.getStats().getHitRatio() * 100)), jamonWriter); // 218, 76 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf("%"), jamonWriter); // 218, 85 jamonWriter.write("</td>\n <td>Hit Count divided by total requests count</td>\n </tr>\n\n"); }
assertNotNull(bc); CacheStats cs = bc.getStats(); long startHit = cs.getHitCount(); long startMiss = cs.getMissCount(); long startEvicted = cs.getEvictedCount(); scanner.seek(KeyValue.LOWESTKEY); while (scanner.next() != null); assertEquals(startHit, cs.getHitCount()); assertEquals(startMiss + 3, cs.getMissCount()); assertEquals(startEvicted, cs.getEvictedCount()); startMiss += 3; scanner.close(); scanner.seek(KeyValue.LOWESTKEY); while (scanner.next() != null); assertEquals(startHit + 3, cs.getHitCount()); assertEquals(startMiss, cs.getMissCount()); assertEquals(startEvicted, cs.getEvictedCount()); startHit += 3; scanner.close(); assertEquals(startHit + 6, cs.getHitCount()); assertEquals(startMiss, cs.getMissCount()); assertEquals(startEvicted, cs.getEvictedCount()); startHit += 6; scannerOne.close(); assertEquals(startHit, cs.getHitCount());
private void __jamon_innerUnit__evictions_tmpl(final java.io.Writer jamonWriter, final BlockCache bc) throws java.io.IOException { // 171, 1 AgeSnapshot ageAtEvictionSnapshot = bc.getStats().getAgeAtEvictionSnapshot(); // Only show if non-zero mean and stddev as is the case in combinedblockcache // 175, 5 jamonWriter.write("<tr>\n <td>Evicted</td>\n <td>"); // 177, 13 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,d", bc.getStats().getEvictedCount())), jamonWriter); // 177, 72 jamonWriter.write("</td>\n <td>The total number of blocks evicted</td>\n </tr>\n <tr>\n <td>Evictions</td>\n <td>"); // 182, 13 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,d", bc.getStats().getEvictionCount())), jamonWriter); // 182, 73 jamonWriter.write("</td>\n <td>The total number of times an eviction has occurred</td>\n </tr>\n <tr>\n <td>Mean</td>\n <td>"); // 187, 13 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,d", (long)ageAtEvictionSnapshot.getMean())), jamonWriter); // 187, 78 jamonWriter.write("</td>\n <td>Mean age of Blocks at eviction time (seconds)</td>\n </tr>\n"); }
@Override public int evictBlocksByHfileName(String hfileName) { stats.evicted(0, true); // Just assuming only one block for file here. return 0; }
@Override public long getEvictionCount() { return lruCacheStats.getEvictionCount() + bucketCacheStats.getEvictionCount(); }
@Override public long getEvictedCount() { return lruCacheStats.getEvictedCount() + bucketCacheStats.getEvictedCount(); }
private void addDataAndHits(final BlockCache bc, final int count) { Cacheable dce = new DataCacheEntry(); Cacheable ice = new IndexCacheEntry(); for (int i = 0; i < count; i++) { BlockCacheKey bckd = new BlockCacheKey("f", i); BlockCacheKey bcki = new BlockCacheKey("f", i + count); bc.getBlock(bckd, true, false, true); bc.cacheBlock(bckd, dce); bc.cacheBlock(bcki, ice); bc.getBlock(bckd, true, false, true); bc.getBlock(bcki, true, false, true); } assertEquals(2 * count /*Data and Index blocks*/, bc.getStats().getHitCount()); BlockCacheKey bckd = new BlockCacheKey("f", 0); BlockCacheKey bcki = new BlockCacheKey("f", 0 + count); bc.evictBlock(bckd); bc.evictBlock(bcki); bc.getStats().getEvictedCount(); }