private static BucketCacheProtos.BlockCacheKey toPB(BlockCacheKey key) { return BucketCacheProtos.BlockCacheKey.newBuilder() .setHfilename(key.getHfileName()) .setOffset(key.getOffset()) .setPrimaryReplicaBlock(key.isPrimary()) .setBlockType(toPB(key.getBlockType())) .build(); }
CachedItem(String blockName, int size) { this.cacheKey = new BlockCacheKey(blockName, 0); this.size = size; }
public LruCachedBlock(BlockCacheKey cacheKey, Cacheable buf, long accessTime, boolean inMemory) { this.cacheKey = cacheKey; this.buf = buf; this.accessTime = accessTime; // We approximate the size of this class by the size of its name string // plus the size of its byte buffer plus the overhead associated with all // the base classes. We also include the base class // sizes in the PER_BLOCK_OVERHEAD variable rather than align()ing them with // their buffer lengths. This variable is used elsewhere in unit tests. this.size = ClassSize.align(cacheKey.heapSize()) + ClassSize.align(buf.heapSize()) + PER_BLOCK_OVERHEAD; if(inMemory) { this.priority = BlockPriority.MEMORY; } else { this.priority = BlockPriority.SINGLE; } }
@Override public int compare(BlockCacheKey a, BlockCacheKey b) { int nameComparison = a.getHfileName().compareTo(b.getHfileName()); if (nameComparison != 0) { return nameComparison; } if (a.getOffset() == b.getOffset()) { return 0; } else if (a.getOffset() < b.getOffset()) { return -1; } return 1; } });
@Override public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, boolean updateCacheMetrics) { // Assume that nothing is the block cache HFileBlock result = null; try (TraceScope traceScope = TraceUtil.createTrace("MemcachedBlockCache.getBlock")) { result = client.get(cacheKey.toString(), tc); } catch (Exception e) { // Catch a pretty broad set of exceptions to limit any changes in the memecache client // and how it handles failures from leaking into the read path. if (LOG.isDebugEnabled()) { LOG.debug("Exception pulling from memcached [ " + cacheKey.toString() + " ]. Treating as a miss.", e); } result = null; } finally { // Update stats if this request doesn't have it turned off 100% of the time if (updateCacheMetrics) { if (result == null) { cacheStats.miss(caching, cacheKey.isPrimary(), cacheKey.getBlockType()); } else { cacheStats.hit(caching, cacheKey.isPrimary(), cacheKey.getBlockType()); } } } return result; }
@Override public boolean evictBlock(BlockCacheKey cacheKey) { stats.evicted(0, cacheKey != null ? cacheKey.isPrimary() : true); return false; }
/** * Used in testing. May be very inefficient. * * @return the set of cached file names */ @VisibleForTesting SortedSet<String> getCachedFileNamesForTest() { SortedSet<String> fileNames = new TreeSet<>(); for (BlockCacheKey cacheKey : map.keySet()) { fileNames.add(cacheKey.getHfileName()); } return fileNames; }
if (cb == null) { if (!repeat && updateCacheMetrics) { stats.miss(caching, cacheKey.isPrimary(), cacheKey.getBlockType()); if (updateCacheMetrics) stats.hit(caching, cacheKey.isPrimary(), cacheKey.getBlockType()); cb.access(count.incrementAndGet()); return cb.getBuffer();
return; int queueNum = (cacheKey.hashCode() & 0x7FFFFFFF) % writerQueues.size(); BlockingQueue<RAMQueueEntry> bq = writerQueues.get(queueNum); boolean successfulAddition = false; this.blockNumber.incrementAndGet(); this.heapSize.addAndGet(cachedItem.heapSize()); blocksByHFile.put(cacheKey.getHfileName(), cacheKey);
@Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { if (buf instanceof HFileBlock) { client.add(cacheKey.toString(), MAX_SIZE, (HFileBlock) buf, tc); } else { if (LOG.isDebugEnabled()) { LOG.debug("MemcachedBlockCache can not cache Cacheable's of type " + buf.getClass().toString()); } } }
return; int queueNum = (cacheKey.hashCode() & 0x7FFFFFFF) % writerQueues.size(); BlockingQueue<RAMQueueEntry> bq = writerQueues.get(queueNum); boolean successfulAddition = false;
private boolean forceEvict(BlockCacheKey cacheKey) { if (!cacheEnabled) { return false; } RAMQueueEntry removedBlock = checkRamCache(cacheKey); BucketEntry bucketEntry = backingMap.get(cacheKey); if (bucketEntry == null) { if (removedBlock != null) { cacheStats.evicted(0, cacheKey.isPrimary()); return true; } else { return false; } } ReentrantReadWriteLock lock = offsetLock.getLock(bucketEntry.offset()); try { lock.writeLock().lock(); if (backingMap.remove(cacheKey, bucketEntry)) { blockEvicted(cacheKey, bucketEntry, removedBlock == null); } else { return false; } } finally { lock.writeLock().unlock(); } cacheStats.evicted(bucketEntry.getCachedTime(), cacheKey.isPrimary()); return true; }
/** * Evicts all blocks for a specific HFile. This is an * expensive operation implemented as a linear-time search through all blocks * in the cache. Ideally this should be a search in a log-access-time map. * * <p> * This is used for evict-on-close to remove all blocks of a specific HFile. * * @return the number of blocks evicted */ @Override public int evictBlocksByHfileName(String hfileName) { int numEvicted = 0; for (BlockCacheKey key : map.keySet()) { if (key.getHfileName().equals(hfileName)) { if (evictBlock(key)) ++numEvicted; } } if (victimHandler != null) { numEvicted += victimHandler.evictBlocksByHfileName(hfileName); } return numEvicted; }
@Override public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, boolean updateCacheMetrics) { // Assume that nothing is the block cache HFileBlock result = null; try (TraceScope traceScope = TraceUtil.createTrace("MemcachedBlockCache.getBlock")) { result = client.get(cacheKey.toString(), tc); } catch (Exception e) { // Catch a pretty broad set of exceptions to limit any changes in the memecache client // and how it handles failures from leaking into the read path. if (LOG.isDebugEnabled()) { LOG.debug("Exception pulling from memcached [ " + cacheKey.toString() + " ]. Treating as a miss.", e); } result = null; } finally { // Update stats if this request doesn't have it turned off 100% of the time if (updateCacheMetrics) { if (result == null) { cacheStats.miss(caching, cacheKey.isPrimary(), cacheKey.getBlockType()); } else { cacheStats.hit(caching, cacheKey.isPrimary(), cacheKey.getBlockType()); } } } return result; }
if (re != null) { if (updateCacheMetrics) { cacheStats.hit(caching, key.isPrimary(), key.getBlockType()); long timeTaken = System.nanoTime() - start; if (updateCacheMetrics) { cacheStats.hit(caching, key.isPrimary(), key.getBlockType()); cacheStats.ioHit(timeTaken); cacheStats.miss(caching, key.isPrimary(), key.getBlockType());
+ cacheKey.getHfileName() + " @ " + cacheKey.getOffset() + " is " + buf.heapSize() + " which is larger than " + maxBlockSize);
@Override public boolean evictBlock(BlockCacheKey cacheKey) { try { cacheStats.evict(); return client.delete(cacheKey.toString()).get(); } catch (InterruptedException e) { LOG.warn("Error deleting " + cacheKey.toString(), e); Thread.currentThread().interrupt(); } catch (ExecutionException e) { if (LOG.isDebugEnabled()) { LOG.debug("Error deleting " + cacheKey.toString(), e); } } return false; }
CachedItem(String blockName, int size, int offset) { this.cacheKey = new BlockCacheKey(blockName, offset); this.size = size; }