public StorefileRefresherChore(int period, boolean onlyMetaRefresh, HRegionServer regionServer, Stoppable stoppable) { super("StorefileRefresherChore", stoppable, period); this.period = period; this.regionServer = regionServer; this.hfileTtl = this.regionServer.getConfiguration().getLong( TimeToLiveHFileCleaner.TTL_CONF_KEY, TimeToLiveHFileCleaner.DEFAULT_TTL); this.onlyMetaRefresh = onlyMetaRefresh; if (period > hfileTtl / 2) { throw new RuntimeException(REGIONSERVER_STOREFILE_REFRESH_PERIOD + " should be set smaller than half of " + TimeToLiveHFileCleaner.TTL_CONF_KEY); } lastRefreshTimes = new HashMap<>(); }
public FileSystemUtilizationChore(HRegionServer rs) { super(FileSystemUtilizationChore.class.getSimpleName(), rs, getPeriod(rs.getConfiguration()), getInitialDelay(rs.getConfiguration()), getTimeUnit(rs.getConfiguration())); this.rs = rs; this.maxIterationMillis = rs.getConfiguration().getLong( FS_UTILIZATION_MAX_ITERATION_DURATION_KEY, FS_UTILIZATION_MAX_ITERATION_DURATION_DEFAULT); }
private void splitWal() throws IOException { SplitLogWorker.TaskExecutor.Status status = SplitLogWorker.splitLog(walPath, null, rs.getConfiguration(), rs, rs, rs.walFactory); if (status != SplitLogWorker.TaskExecutor.Status.DONE) { throw new IOException("Split WAL " + walPath + " failed at server "); } } }
@VisibleForTesting public Configuration getConfiguration() { return regionServer.getConfiguration(); }
private HRegionServer mockRegionServer(Configuration conf) { HRegionServer rs = mock(HRegionServer.class); when(rs.getConfiguration()).thenReturn(conf); return rs; } }
/** @param server */ CompactSplit(HRegionServer server) { this.server = server; this.conf = server.getConfiguration(); this.compactionsEnabled = this.conf.getBoolean(HBASE_REGION_SERVER_ENABLE_COMPACTION,true); createCompactionExecutors(); createSplitExcecutors(); // compaction throughput controller this.compactionThroughputController = CompactionThroughputControllerFactory.create(server, conf); }
@Override public void init(byte[] parameter, HRegionServer rs) { this.rs = rs; this.fs = rs.getWALFileSystem(); this.conf = rs.getConfiguration(); try { ReplaySyncReplicationWALParameter param = ReplaySyncReplicationWALParameter.parseFrom(parameter); this.peerId = param.getPeerId(); param.getWalList().forEach(this.wals::add); this.batchSize = rs.getConfiguration().getLong(REPLAY_SYNC_REPLICATION_WAL_BATCH_SIZE, DEFAULT_REPLAY_SYNC_REPLICATION_WAL_BATCH_SIZE); } catch (InvalidProtocolBufferException e) { initError = e; } }
private Reader getReader(String wal) throws IOException { Path path = new Path(rs.getWALRootDir(), wal); long length = rs.getWALFileSystem().getFileStatus(path).getLen(); try { FSUtils.getInstance(fs, conf).recoverFileLease(fs, path, conf); return WALFactory.createReader(rs.getWALFileSystem(), path, rs.getConfiguration()); } catch (EOFException e) { if (length <= 0) { LOG.warn("File is empty. Could not open {} for reading because {}", path, e); return null; } throw e; } }
public MetricsRegionServerWrapperImpl(final HRegionServer regionServer) { this.regionServer = regionServer; initBlockCache(); initMobFileCache(); this.period = regionServer.conf.getLong(HConstants.REGIONSERVER_METRICS_PERIOD, HConstants.DEFAULT_REGIONSERVER_METRICS_PERIOD); this.executor = CompatibilitySingletonFactory.getInstance(MetricsExecutor.class).getExecutor(); this.runnable = new RegionServerMetricsWrapperRunnable(); this.executor.scheduleWithFixedDelay(this.runnable, this.period, this.period, TimeUnit.MILLISECONDS); this.metricsWALSource = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class); try { this.dfsHedgedReadMetrics = FSUtils.getDFSHedgedReadMetrics(regionServer.getConfiguration()); } catch (IOException e) { LOG.warn("Failed to get hedged metrics", e); } if (LOG.isInfoEnabled()) { LOG.info("Computing regionserver metrics every " + this.period + " milliseconds"); } }
/** * Creates an HRegionServer using the given Configuration. */ private HRegionServer mockRegionServer(Configuration conf) { final HRegionServer rs = mock(HRegionServer.class); final RegionServerSpaceQuotaManager quotaManager = mock(RegionServerSpaceQuotaManager.class); when(rs.getConfiguration()).thenReturn(conf); when(rs.getRegionServerSpaceQuotaManager()).thenReturn(quotaManager); when(quotaManager.getRegionSizeStore()).thenReturn(new RegionSizeStoreImpl()); return rs; }
@Before public void setupBasicMocks() throws IOException, ServiceException { rs = Mockito.mock(HRegionServer.class); rpcServices = Mockito.mock(RSRpcServices.class); rpcServer = Mockito.mock(RpcServerInterface.class); Mockito.doReturn(HBaseConfiguration.create()) .when(rs).getConfiguration(); Mockito.doReturn(rpcServices).when(rs).getRSRpcServices(); Mockito.doReturn(rpcServer).when(rs).getRpcServer(); Mockito.doReturn(fakeResponse).when(rpcServices).getServerInfo( (RpcController)Mockito.any(), (GetServerInfoRequest)Mockito.any()); // Fake ZKW ZKWatcher zkw = Mockito.mock(ZKWatcher.class); Mockito.doReturn("fakequorum").when(zkw).getQuorum(); Mockito.doReturn(zkw).when(rs).getZooKeeper(); // Fake BlockCache LOG.warn("The " + HConstants.HFILE_BLOCK_CACHE_SIZE_KEY + " is set to 0"); Mockito.doReturn(Optional.empty()).when(rs).getBlockCache(); // Fake MasterAddressTracker MasterAddressTracker mat = Mockito.mock(MasterAddressTracker.class); Mockito.doReturn(fakeMasterAddress).when(mat).getMasterAddress(); Mockito.doReturn(mat).when(rs).getMasterAddressTracker(); MetricsRegionServer rms = Mockito.mock(MetricsRegionServer.class); Mockito.doReturn(new MetricsRegionServerWrapperStub()).when(rms).getRegionServerWrapper(); Mockito.doReturn(rms).when(rs).getRegionServerMetrics(); MetricsHBaseServer ms = Mockito.mock(MetricsHBaseServer.class); Mockito.doReturn(new MetricsHBaseServerWrapperStub()).when(ms).getHBaseServerWrapper(); Mockito.doReturn(ms).when(rpcServer).getMetrics(); }
@Test public void testRegionServerOnlineConfigChange() throws Exception { replaceHBaseSiteXML(); admin.getRegionServers().get().forEach(server -> admin.updateConfiguration(server).join()); // Check the configuration of the RegionServers TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().forEach(thread -> { Configuration conf = thread.getRegionServer().getConfiguration(); assertEquals(1000, conf.getInt("hbase.custom.config", 0)); }); restoreHBaseSiteXML(); }
TEST_UTIL.getMiniHBaseCluster().getRegionServer(0).getConfiguration(); custom = regionServerConfiguration.getInt("hbase.custom.config", 0); assertEquals(1000, custom);
/** * Create a custom compaction request and be sure that we can track it through the queue, knowing * when the compaction is completed. */ @Test public void testTrackingCompactionRequest() throws Exception { // setup a compact/split thread on a mock server HRegionServer mockServer = Mockito.mock(HRegionServer.class); Mockito.when(mockServer.getConfiguration()).thenReturn(r.getBaseConf()); CompactSplit thread = new CompactSplit(mockServer); Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread); // setup a region/store with some files HStore store = r.getStore(COLUMN_FAMILY); createStoreFile(r); for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) { createStoreFile(r); } CountDownLatch latch = new CountDownLatch(1); Tracker tracker = new Tracker(latch); thread.requestCompaction(r, store, "test custom comapction", PRIORITY_USER, tracker, null); // wait for the latch to complete. latch.await(); thread.interruptIfNecessary(); }
@Test public void testAllClusterOnlineConfigChange() throws IOException { replaceHBaseSiteXML(); admin.updateConfiguration().join(); // Check the configuration of the Masters TEST_UTIL.getMiniHBaseCluster().getMasterThreads().forEach(thread -> { Configuration conf = thread.getMaster().getConfiguration(); assertEquals(1000, conf.getInt("hbase.custom.config", 0)); }); // Check the configuration of the RegionServers TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().forEach(thread -> { Configuration conf = thread.getRegionServer().getConfiguration(); assertEquals(1000, conf.getInt("hbase.custom.config", 0)); }); restoreHBaseSiteXML(); }
WaitThroughPutController.class.getName()); HRegionServer mockServer = Mockito.mock(HRegionServer.class); Mockito.when(mockServer.getConfiguration()).thenReturn(r.getBaseConf()); CompactSplit thread = new CompactSplit(mockServer);
Mockito.when(mockServer.getConfiguration()).thenReturn(r.getBaseConf()); CompactSplit thread = new CompactSplit(mockServer); Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread);
/** * HBASE-7947: Regression test to ensure adding to the correct list in the * {@link CompactSplit} * @throws Exception on failure */ @Test public void testMultipleCustomCompactionRequests() throws Exception { // setup a compact/split thread on a mock server HRegionServer mockServer = Mockito.mock(HRegionServer.class); Mockito.when(mockServer.getConfiguration()).thenReturn(r.getBaseConf()); CompactSplit thread = new CompactSplit(mockServer); Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread); // setup a region/store with some files int numStores = r.getStores().size(); CountDownLatch latch = new CountDownLatch(numStores); Tracker tracker = new Tracker(latch); // create some store files and setup requests for each store on which we want to do a // compaction for (HStore store : r.getStores()) { createStoreFile(r, store.getColumnFamilyName()); createStoreFile(r, store.getColumnFamilyName()); createStoreFile(r, store.getColumnFamilyName()); thread.requestCompaction(r, store, "test mulitple custom comapctions", PRIORITY_USER, tracker, null); } // wait for the latch to complete. latch.await(); thread.interruptIfNecessary(); }
/** * Test no new Compaction requests are generated after calling stop compactions */ @Test public void testStopStartCompaction() throws IOException { // setup a compact/split thread on a mock server HRegionServer mockServer = Mockito.mock(HRegionServer.class); Mockito.when(mockServer.getConfiguration()).thenReturn(r.getBaseConf()); CompactSplit thread = new CompactSplit(mockServer); Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread); // setup a region/store with some files HStore store = r.getStore(COLUMN_FAMILY); createStoreFile(r); for (int i = 0; i < HStore.DEFAULT_BLOCKING_STOREFILE_COUNT - 1; i++) { createStoreFile(r); } thread.switchCompaction(false); thread .requestCompaction(r, store, "test", Store.PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null); assertEquals(false, thread.isCompactionsEnabled()); assertEquals(0, thread.getLongCompactions().getActiveCount() + thread.getShortCompactions() .getActiveCount()); thread.switchCompaction(true); assertEquals(true, thread.isCompactionsEnabled()); thread .requestCompaction(r, store, "test", Store.PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null); assertEquals(1, thread.getLongCompactions().getActiveCount() + thread.getShortCompactions() .getActiveCount()); }
@Test public void testRemoveClosedWAL() throws Exception { HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); Configuration conf = rs.getConfiguration(); LogRoller logRoller = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0).getWalRoller(); int originalSize = logRoller.getWalNeedsRoll().size();