/** * Compares the actual type of a block retrieved from cache or disk with its * expected type and throws an exception in case of a mismatch. Expected * block type of {@link BlockType#DATA} is considered to match the actual * block type [@link {@link BlockType#ENCODED_DATA} as well. * @param block a block retrieved from cache or disk * @param expectedBlockType the expected block type, or null to skip the * check */ private void validateBlockType(HFileBlock block, BlockType expectedBlockType) throws IOException { if (expectedBlockType == null) { return; } BlockType actualBlockType = block.getBlockType(); if (expectedBlockType.isData() && actualBlockType.isData()) { // We consider DATA to match ENCODED_DATA for the purpose of this // verification. return; } if (actualBlockType != expectedBlockType) { throw new IOException("Expected block type " + expectedBlockType + ", " + "but got " + actualBlockType + ": " + block); } }
/** * Helper function that updates the local size counter and also updates any * per-cf or per-blocktype metrics it can discern from given * {@link LruCachedBlock} */ private long updateSizeMetrics(LruCachedBlock cb, boolean evict) { long heapsize = cb.heapSize(); BlockType bt = cb.getBuffer().getBlockType(); if (evict) { heapsize *= -1; } if (bt != null && bt.isData()) { dataBlockSize.add(heapsize); } return size.addAndGet(heapsize); }
/** * @param filename * @param blocks * @return A JSON String of <code>filename</code> and counts of <code>blocks</code> * @throws JsonGenerationException * @throws JsonMappingException * @throws IOException */ public static String toJSON(final String filename, final NavigableSet<CachedBlock> blocks) throws JsonGenerationException, JsonMappingException, IOException { CachedBlockCountsPerFile counts = new CachedBlockCountsPerFile(filename); for (CachedBlock cb: blocks) { counts.count++; counts.size += cb.getSize(); BlockType bt = cb.getBlockType(); if (bt != null && bt.isData()) { counts.countData++; counts.sizeData += cb.getSize(); } } return MAPPER.writeValueAsString(counts); }
/** * @param cb * @return True if full.... if we won't be adding any more. */ public boolean update(final CachedBlock cb) { if (isFull()) return true; NavigableSet<CachedBlock> set = this.cachedBlockByFile.get(cb.getFilename()); if (set == null) { set = new ConcurrentSkipListSet<>(); this.cachedBlockByFile.put(cb.getFilename(), set); } set.add(cb); this.size += cb.getSize(); this.count++; BlockType bt = cb.getBlockType(); if (bt != null && bt.isData()) { this.dataBlockCount++; this.dataSize += cb.getSize(); } long age = (this.now - cb.getCachedTime())/NANOS_PER_SECOND; this.hist.add(age, 1); return false; }
private void logPerFile(final BlockCacheUtil.CachedBlocksByFile cbsbf) throws IOException { for (Map.Entry<String, NavigableSet<CachedBlock>> e: cbsbf.getCachedBlockStatsByFile().entrySet()) { int count = 0; long size = 0; int countData = 0; long sizeData = 0; for (CachedBlock cb: e.getValue()) { count++; size += cb.getSize(); BlockType bt = cb.getBlockType(); if (bt != null && bt.isData()) { countData++; sizeData += cb.getSize(); } } LOG.info("filename=" + e.getKey() + ", count=" + count + ", countData=" + countData + ", size=" + size + ", sizeData=" + sizeData); LOG.info(BlockCacheUtil.toJSON(e.getKey(), e.getValue())); } }
map.put(cacheKey, cb); long val = elements.incrementAndGet(); if (buf.getBlockType().isData()) { dataBlockElements.increment();
assertCounterSanity(size, val); if (block.getBuffer().getBlockType().isData()) { dataBlockElements.decrement();
if (block.getBlockType().isData()) { dataBlock = true; break;
if (cachedBlock.getBlockType().isData() && !actualDataBlockEncoding.equals(expectedDataBlockEncoding)) {
public BlockCacheKey(String file, long offset, DataBlockEncoding encoding, BlockType blockType) { this.hfileName = file; this.offset = offset; // We add encoding to the cache key only for data blocks. If the block type // is unknown (this should never be the case in production), we just use // the provided encoding, because it might be a data block. this.encoding = (blockType == null || blockType.isData()) ? encoding : DataBlockEncoding.NONE; }
/** * Compares the actual type of a block retrieved from cache or disk with its * expected type and throws an exception in case of a mismatch. Expected * block type of {@link BlockType#DATA} is considered to match the actual * block type [@link {@link BlockType#ENCODED_DATA} as well. * @param block a block retrieved from cache or disk * @param expectedBlockType the expected block type, or null to skip the * check */ private void validateBlockType(HFileBlock block, BlockType expectedBlockType) throws IOException { if (expectedBlockType == null) { return; } BlockType actualBlockType = block.getBlockType(); if (expectedBlockType.isData() && actualBlockType.isData()) { // We consider DATA to match ENCODED_DATA for the purpose of this // verification. return; } if (actualBlockType != expectedBlockType) { throw new IOException("Expected block type " + expectedBlockType + ", " + "but got " + actualBlockType + ": " + block); } }
/** * @param cb * @return True if full.... if we won't be adding any more. */ public boolean update(final CachedBlock cb) { if (isFull()) return true; NavigableSet<CachedBlock> set = this.cachedBlockByFile.get(cb.getFilename()); if (set == null) { set = new ConcurrentSkipListSet<CachedBlock>(); this.cachedBlockByFile.put(cb.getFilename(), set); } set.add(cb); this.size += cb.getSize(); this.count++; BlockType bt = cb.getBlockType(); if (bt != null && bt.isData()) { this.dataBlockCount++; this.dataSize += cb.getSize(); } long age = this.now - cb.getCachedTime(); this.age.update(age); return false; }
private void logPerFile(final BlockCacheUtil.CachedBlocksByFile cbsbf) throws IOException { for (Map.Entry<String, NavigableSet<CachedBlock>> e: cbsbf.getCachedBlockStatsByFile().entrySet()) { int count = 0; long size = 0; int countData = 0; long sizeData = 0; for (CachedBlock cb: e.getValue()) { count++; size += cb.getSize(); BlockType bt = cb.getBlockType(); if (bt != null && bt.isData()) { countData++; sizeData += cb.getSize(); } } LOG.info("filename=" + e.getKey() + ", count=" + count + ", countData=" + countData + ", size=" + size + ", sizeData=" + sizeData); LOG.info(BlockCacheUtil.toJSON(e.getKey(), e.getValue())); } }
/** * @param filename * @param blocks * @return A JSON String of <code>filename</code> and counts of <code>blocks</code> * @throws JsonGenerationException * @throws JsonMappingException * @throws IOException */ public static String toJSON(final String filename, final NavigableSet<CachedBlock> blocks) throws JsonGenerationException, JsonMappingException, IOException { CachedBlockCountsPerFile counts = new CachedBlockCountsPerFile(filename); for (CachedBlock cb: blocks) { counts.count++; counts.size += cb.getSize(); BlockType bt = cb.getBlockType(); if (bt != null && bt.isData()) { counts.countData++; counts.sizeData += cb.getSize(); } } return MAPPER.writeValueAsString(counts); }
/** * Scans blocks in the "scanned" section of the {@link HFile} until the next * data block is found. * * @return the next block, or null if there are no more data blocks * @throws IOException */ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH", justification="Yeah, unnecessary null check; could do w/ clean up") protected HFileBlock readNextDataBlock() throws IOException { long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset(); if (block == null) return null; HFileBlock curBlock = block; do { if (curBlock.getOffset() >= lastDataBlockOffset) { return null; } if (curBlock.getOffset() < 0) { throw new IOException("Invalid block file offset: " + block); } // We are reading the next block without block type validation, because // it might turn out to be a non-data block. curBlock = reader.readBlock(curBlock.getOffset() + curBlock.getOnDiskSizeWithHeader(), curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread, isCompaction, true, null, getEffectiveDataBlockEncoding()); } while (!curBlock.getBlockType().isData()); return curBlock; }
if (block.getBlockType().isData()) { break;
if (cachedBlock.getBlockType().isData() && !actualDataBlockEncoding.equals(expectedDataBlockEncoding)) {