/** * {@inheritDoc} */ @Override public CacheEvictionStats clearBlockCache(final TableName tableName) throws IOException { checkTableExists(tableName); CacheEvictionStatsBuilder cacheEvictionStats = CacheEvictionStats.builder(); List<Pair<RegionInfo, ServerName>> pairs = MetaTableAccessor.getTableRegionsAndLocations(connection, tableName); Map<ServerName, List<RegionInfo>> regionInfoByServerName = pairs.stream() .filter(pair -> !(pair.getFirst().isOffline())) .filter(pair -> pair.getSecond() != null) .collect(Collectors.groupingBy(pair -> pair.getSecond(), Collectors.mapping(pair -> pair.getFirst(), Collectors.toList()))); for (Map.Entry<ServerName, List<RegionInfo>> entry : regionInfoByServerName.entrySet()) { CacheEvictionStats stats = clearBlockCache(entry.getKey(), entry.getValue()); cacheEvictionStats = cacheEvictionStats.append(stats); if (stats.getExceptionCount() > 0) { for (Map.Entry<byte[], Throwable> exception : stats.getExceptions().entrySet()) { LOG.debug("Failed to clear block cache for " + Bytes.toStringBinary(exception.getKey()) + " on " + entry.getKey() + ": ", exception.getValue()); } } } return cacheEvictionStats.build(); }
public CacheEvictionStatsBuilder append(CacheEvictionStats stats) { this.evictedBlocks += stats.getEvictedBlocks(); this.maxCacheSize += stats.getMaxCacheSize(); this.exceptions.putAll(stats.getExceptions()); return this; }
public CacheEvictionStats build() { return new CacheEvictionStats(this); } }
@Override public String toString() { return "CacheEvictionStats{" + "evictedBlocks=" + evictedBlocks + ", maxCacheSize=" + maxCacheSize + ", failedRegionsSize=" + getExceptionCount() + ", failedRegions=" + getFailedRegions() + '}'; } }
public CacheEvictionStats clearRegionBlockCache(Region region) { long evictedBlocks = 0; for(Store store : region.getStores()) { for(StoreFile hFile : store.getStorefiles()) { evictedBlocks += blockCache.evictBlocksByHfileName(hFile.getPath().getName()); } } return CacheEvictionStats.builder() .withEvictedBlocks(evictedBlocks) .build(); }
@Test public void testClearBlockCacheFromAsyncAdmin() throws Exception { AsyncAdmin admin = ConnectionFactory.createAsyncConnection(HTU.getConfiguration()).get().getAdmin(); BlockCache blockCache1 = rs1.getBlockCache().get(); BlockCache blockCache2 = rs2.getBlockCache().get(); long initialBlockCount1 = blockCache1.getBlockCount(); long initialBlockCount2 = blockCache2.getBlockCount(); // scan will cause blocks to be added in BlockCache scanAllRegionsForRS(rs1); assertEquals(blockCache1.getBlockCount() - initialBlockCount1, HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY)); scanAllRegionsForRS(rs2); assertEquals(blockCache2.getBlockCount() - initialBlockCount2, HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY)); CacheEvictionStats stats = admin.clearBlockCache(TABLE_NAME).get(); assertEquals(stats.getEvictedBlocks(), HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY) + HTU .getNumHFilesForRS(rs2, TABLE_NAME, FAMILY)); assertEquals(initialBlockCount1, blockCache1.getBlockCount()); assertEquals(initialBlockCount2, blockCache2.getBlockCount()); }
@Override public String toString() { return "CacheEvictionStats{" + "evictedBlocks=" + evictedBlocks + ", maxCacheSize=" + maxCacheSize + ", failedRegionsSize=" + getExceptionCount() + ", failedRegions=" + getFailedRegions() + '}'; } }
@Override public ClearRegionBlockCacheResponse clearRegionBlockCache(RpcController controller, ClearRegionBlockCacheRequest request) { ClearRegionBlockCacheResponse.Builder builder = ClearRegionBlockCacheResponse.newBuilder(); CacheEvictionStatsBuilder stats = CacheEvictionStats.builder(); List<HRegion> regions = getRegions(request.getRegionList(), stats); for (HRegion region : regions) { try { stats = stats.append(this.regionServer.clearRegionBlockCache(region)); } catch (Exception e) { stats.addException(region.getRegionInfo().getRegionName(), e); } } stats.withMaxCacheSize(regionServer.getBlockCache().map(BlockCache::getMaxSize).orElse(0L)); return builder.setStats(ProtobufUtil.toCacheEvictionStats(stats.build())).build(); }
@Test public void testClearBlockCacheFromAdmin() throws Exception { Admin admin = HTU.getAdmin(); BlockCache blockCache1 = rs1.getBlockCache().get(); BlockCache blockCache2 = rs2.getBlockCache().get(); long initialBlockCount1 = blockCache1.getBlockCount(); long initialBlockCount2 = blockCache2.getBlockCount(); // scan will cause blocks to be added in BlockCache scanAllRegionsForRS(rs1); assertEquals(blockCache1.getBlockCount() - initialBlockCount1, HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY)); scanAllRegionsForRS(rs2); assertEquals(blockCache2.getBlockCount() - initialBlockCount2, HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY)); CacheEvictionStats stats = admin.clearBlockCache(TABLE_NAME); assertEquals(stats.getEvictedBlocks(), HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY) + HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY)); assertEquals(initialBlockCount1, blockCache1.getBlockCount()); assertEquals(initialBlockCount2, blockCache2.getBlockCount()); }
public static HBaseProtos.CacheEvictionStats toCacheEvictionStats( CacheEvictionStats cacheEvictionStats) { HBaseProtos.CacheEvictionStats.Builder builder = HBaseProtos.CacheEvictionStats.newBuilder(); for (Map.Entry<byte[], Throwable> entry : cacheEvictionStats.getExceptions().entrySet()) { builder.addException( RegionExceptionMessage.newBuilder() .setRegion(RequestConverter.buildRegionSpecifier( RegionSpecifierType.REGION_NAME, entry.getKey())) .setException(ResponseConverter.buildException(entry.getValue())) .build() ); } return builder .setEvictedBlocks(cacheEvictionStats.getEvictedBlocks()) .setMaxCacheSize(cacheEvictionStats.getMaxCacheSize()) .build(); }
/** * {@inheritDoc} */ @Override public CacheEvictionStats clearBlockCache(final TableName tableName) throws IOException { checkTableExists(tableName); CacheEvictionStatsBuilder cacheEvictionStats = CacheEvictionStats.builder(); List<Pair<RegionInfo, ServerName>> pairs = MetaTableAccessor.getTableRegionsAndLocations(connection, tableName); Map<ServerName, List<RegionInfo>> regionInfoByServerName = pairs.stream() .filter(pair -> !(pair.getFirst().isOffline())) .filter(pair -> pair.getSecond() != null) .collect(Collectors.groupingBy(pair -> pair.getSecond(), Collectors.mapping(pair -> pair.getFirst(), Collectors.toList()))); for (Map.Entry<ServerName, List<RegionInfo>> entry : regionInfoByServerName.entrySet()) { CacheEvictionStats stats = clearBlockCache(entry.getKey(), entry.getValue()); cacheEvictionStats = cacheEvictionStats.append(stats); if (stats.getExceptionCount() > 0) { for (Map.Entry<byte[], Throwable> exception : stats.getExceptions().entrySet()) { LOG.debug("Failed to clear block cache for " + Bytes.toStringBinary(exception.getKey()) + " on " + entry.getKey() + ": ", exception.getValue()); } } } return cacheEvictionStats.build(); }
@Override public String toString() { return "CacheEvictionStats{" + "evictedBlocks=" + evictedBlocks + ", maxCacheSize=" + maxCacheSize + ", failedRegionsSize=" + getExceptionCount() + ", failedRegions=" + getFailedRegions() + '}'; } }
public static CacheEvictionStats toCacheEvictionStats( HBaseProtos.CacheEvictionStats stats) throws IOException{ CacheEvictionStatsBuilder builder = CacheEvictionStats.builder(); builder.withEvictedBlocks(stats.getEvictedBlocks()) .withMaxCacheSize(stats.getMaxCacheSize()); if (stats.getExceptionCount() > 0) { for (HBaseProtos.RegionExceptionMessage exception : stats.getExceptionList()) { HBaseProtos.RegionSpecifier rs = exception.getRegion(); byte[] regionName = rs.getValue().toByteArray(); builder.addException(regionName, ProtobufUtil.toException(exception.getException())); } } return builder.build(); }
@Test public void testClearBlockCacheFromAsyncAdmin() throws Exception { AsyncAdmin admin = ConnectionFactory.createAsyncConnection(HTU.getConfiguration()).get().getAdmin(); // All RS run in a same process, so the block cache is same for rs1 and rs2 BlockCache blockCache = rs1.getCacheConfig().getBlockCache(); long initialBlockCount = blockCache.getBlockCount(); // scan will cause blocks to be added in BlockCache scanAllRegionsForRS(rs1); assertEquals(blockCache.getBlockCount() - initialBlockCount, HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY)); scanAllRegionsForRS(rs2); assertEquals(blockCache.getBlockCount() - initialBlockCount, HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY) + HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY)); CacheEvictionStats stats = admin.clearBlockCache(TABLE_NAME).get(); assertEquals(stats.getEvictedBlocks(), HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY) + HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY)); assertEquals(initialBlockCount, blockCache.getBlockCount()); }
public CacheEvictionStats build() { return new CacheEvictionStats(this); } }
public CacheEvictionStatsBuilder append(CacheEvictionStats stats) { this.evictedBlocks += stats.getEvictedBlocks(); this.maxCacheSize += stats.getMaxCacheSize(); this.exceptions.putAll(stats.getExceptions()); return this; }
/** * {@inheritDoc} */ @Override public CacheEvictionStats clearBlockCache(final TableName tableName) throws IOException { checkTableExists(tableName); CacheEvictionStatsBuilder cacheEvictionStats = CacheEvictionStats.builder(); List<Pair<RegionInfo, ServerName>> pairs = MetaTableAccessor.getTableRegionsAndLocations(connection, tableName); Map<ServerName, List<RegionInfo>> regionInfoByServerName = pairs.stream() .filter(pair -> !(pair.getFirst().isOffline())) .filter(pair -> pair.getSecond() != null) .collect(Collectors.groupingBy(pair -> pair.getSecond(), Collectors.mapping(pair -> pair.getFirst(), Collectors.toList()))); for (Map.Entry<ServerName, List<RegionInfo>> entry : regionInfoByServerName.entrySet()) { CacheEvictionStats stats = clearBlockCache(entry.getKey(), entry.getValue()); cacheEvictionStats = cacheEvictionStats.append(stats); if (stats.getExceptionCount() > 0) { for (Map.Entry<byte[], Throwable> exception : stats.getExceptions().entrySet()) { LOG.debug("Failed to clear block cache for " + Bytes.toStringBinary(exception.getKey()) + " on " + entry.getKey() + ": ", exception.getValue()); } } } return cacheEvictionStats.build(); }
public static CacheEvictionStats toCacheEvictionStats( HBaseProtos.CacheEvictionStats stats) throws IOException{ CacheEvictionStatsBuilder builder = CacheEvictionStats.builder(); builder.withEvictedBlocks(stats.getEvictedBlocks()) .withMaxCacheSize(stats.getMaxCacheSize()); if (stats.getExceptionCount() > 0) { for (HBaseProtos.RegionExceptionMessage exception : stats.getExceptionList()) { HBaseProtos.RegionSpecifier rs = exception.getRegion(); byte[] regionName = rs.getValue().toByteArray(); builder.addException(regionName, ProtobufUtil.toException(exception.getException())); } } return builder.build(); }
@Test public void testClearBlockCacheFromAdmin() throws Exception { Admin admin = HTU.getAdmin(); // All RS run in a same process, so the block cache is same for rs1 and rs2 BlockCache blockCache = rs1.getCacheConfig().getBlockCache(); long initialBlockCount = blockCache.getBlockCount(); // scan will cause blocks to be added in BlockCache scanAllRegionsForRS(rs1); assertEquals(blockCache.getBlockCount() - initialBlockCount, HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY)); scanAllRegionsForRS(rs2); assertEquals(blockCache.getBlockCount() - initialBlockCount, HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY) + HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY)); CacheEvictionStats stats = admin.clearBlockCache(TABLE_NAME); assertEquals(stats.getEvictedBlocks(), HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY) + HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY)); assertEquals(initialBlockCount, blockCache.getBlockCount()); }
public CacheEvictionStats build() { return new CacheEvictionStats(this); } }