public AssignmentManager getAssignmentManager() { return master.getAssignmentManager(); }
private TableDescriptor loadTableDescriptor() throws FileNotFoundException, IOException { TableDescriptor htd = this.master.getTableDescriptors().get(snapshotTable); if (htd == null) { throw new IOException("TableDescriptor missing for " + snapshotTable); } return htd; }
@Override public void initialize(MasterServices master, MetricsMaster metricsMaster) throws KeeperException, IOException, UnsupportedOperationException { this.master = master; this.rootDir = master.getMasterFileSystem().getRootDir(); checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem()); // get the configuration for the coordinator Configuration conf = master.getConfiguration(); long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT); long timeoutMillis = Math.max(conf.getLong(SnapshotDescriptionUtils.SNAPSHOT_TIMEOUT_MILLIS_KEY, SnapshotDescriptionUtils.SNAPSHOT_TIMEOUT_MILLIS_DEFAULT), conf.getLong(SnapshotDescriptionUtils.MASTER_SNAPSHOT_TIMEOUT_MILLIS, SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME)); int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT); // setup the default procedure coordinator String name = master.getServerName().toString(); ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads); ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinator( master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name); this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency); this.executorService = master.getExecutorService(); resetTempDir(); snapshotHandlerChoreCleanerTask = scheduleThreadPool.scheduleAtFixedRate(this::cleanupSentinels, 10, 10, TimeUnit.SECONDS); }
CatalogJanitor(final MasterServices services) { super("CatalogJanitor-" + services.getServerName().toShortString(), services, services.getConfiguration().getInt("hbase.catalogjanitor.interval", 300000)); this.services = services; this.connection = services.getConnection(); }
private RSGroupInfoManagerImpl(MasterServices masterServices) throws IOException { this.masterServices = masterServices; this.watcher = masterServices.getZooKeeper(); this.conn = masterServices.getConnection(); this.rsGroupStartupWorker = new RSGroupStartupWorker(); }
@Override public void setMasterServices(MasterServices masterServices) { masterServerName = masterServices.getServerName(); this.services = masterServices; if (useRegionFinder) { this.regionFinder.setServices(masterServices); } if (this.services.isInMaintenanceMode()) { this.maintenanceMode = true; } }
@Override public void initialize(MasterServices master, MetricsMaster metricsMaster) throws KeeperException, IOException, UnsupportedOperationException { this.master = master; // get the configuration for the coordinator Configuration conf = master.getConfiguration(); long wakeFrequency = conf.getInt(FLUSH_WAKE_MILLIS_KEY, FLUSH_WAKE_MILLIS_DEFAULT); long timeoutMillis = conf.getLong(FLUSH_TIMEOUT_MILLIS_KEY, FLUSH_TIMEOUT_MILLIS_DEFAULT); int threads = conf.getInt(FLUSH_PROC_POOL_THREADS_KEY, FLUSH_PROC_POOL_THREADS_DEFAULT); // setup the procedure coordinator String name = master.getServerName().toString(); ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, threads); ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinator( master.getZooKeeper(), getProcedureSignature(), name); this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency); }
public void start() throws IOException { // If the user doesn't want the quota support skip all the initializations. if (!QuotaUtil.isQuotaEnabled(masterServices.getConfiguration())) { LOG.info("Quota support disabled"); return; } // Create the quota table if missing if (!MetaTableAccessor.tableExists(masterServices.getConnection(), QuotaUtil.QUOTA_TABLE_NAME)) { LOG.info("Quota table not found. Creating..."); createQuotaTable(); } LOG.info("Initializing quota support"); namespaceLocks = new NamedLock<>(); tableLocks = new NamedLock<>(); userLocks = new NamedLock<>(); regionServerLocks = new NamedLock<>(); regionSizes = new ConcurrentHashMap<>(); namespaceQuotaManager = new NamespaceAuditor(masterServices); namespaceQuotaManager.start(); initialized = true; rpcThrottleStorage = new RpcThrottleStorage(masterServices.getZooKeeper(), masterServices.getConfiguration()); }
public MasterWalManager(MasterServices services) throws IOException { this(services.getConfiguration(), services.getMasterFileSystem().getWALFileSystem(), services.getMasterFileSystem().getWALRootDir(), services); }
@Override protected void chore() { try { AssignmentManager am = this.services.getAssignmentManager(); if (this.enabled.get() && !this.services.isInMaintenanceMode() && !this.services.getServerManager().isClusterShutdown() && am != null && am.isMetaLoaded() && !am.hasRegionsInTransition()) { scan(); } else { LOG.warn("CatalogJanitor is disabled! Enabled=" + this.enabled.get() + ", maintenanceMode=" + this.services.isInMaintenanceMode() + ", am=" + am + ", metaLoaded=" + (am != null && am.isMetaLoaded()) + ", hasRIT=" + (am != null && am.hasRegionsInTransition()) + " clusterShutDown=" + this.services .getServerManager().isClusterShutdown()); } } catch (IOException e) { LOG.warn("Failed scan of catalog table", e); } }
public synchronized Optional<ServerName> acquire() { List<ServerName> serverList = master.getServerManager().getOnlineServersList(); Collections.shuffle(serverList); Optional<ServerName> worker = serverList.stream().filter( serverName -> !currentWorkers.containsKey(serverName) || currentWorkers.get(serverName) > 0) .findAny(); if (worker.isPresent()) { currentWorkers.compute(worker.get(), (serverName, availableWorker) -> availableWorker == null ? maxSplitTasks - 1 : availableWorker - 1); } return worker; }
protected static MasterServices getMockedMaster() throws IOException { TableDescriptors tds = Mockito.mock(TableDescriptors.class); Mockito.when(tds.get(tables[0])).thenReturn(tableDescs.get(0)); Mockito.when(tds.get(tables[1])).thenReturn(tableDescs.get(1)); Mockito.when(tds.get(tables[2])).thenReturn(tableDescs.get(2)); Mockito.when(tds.get(tables[3])).thenReturn(tableDescs.get(3)); MasterServices services = Mockito.mock(HMaster.class); Mockito.when(services.getTableDescriptors()).thenReturn(tds); AssignmentManager am = Mockito.mock(AssignmentManager.class); Mockito.when(services.getAssignmentManager()).thenReturn(am); return services; }
private long getRegionSize(RegionInfo hri) { ServerName sn = masterServices.getAssignmentManager().getRegionStates(). getRegionServerOfRegion(hri); RegionMetrics regionLoad = masterServices.getServerManager().getLoad(sn). getRegionMetrics().get(hri.getRegionName()); if (regionLoad == null) { LOG.debug(hri.getRegionNameAsString() + " was not found in RegionsLoad"); return -1; } return (long) regionLoad.getStoreFileSize().get(Size.Unit.MEGABYTE); } }
public RSProcedureDispatcher(final MasterServices master) { super(master.getConfiguration()); this.master = master; this.rsStartupWaitTime = master.getConfiguration().getLong( RS_RPC_STARTUP_WAIT_TIME_CONF_KEY, DEFAULT_RS_RPC_STARTUP_WAIT_TIME); }
/** * @return server name */ public ServerName getServerName() { return master.getServerName(); } }
private void updateMetaLocation(RegionInfo regionInfo, ServerName serverName, State state) throws IOException { try { MetaTableLocator.setMetaLocation(master.getZooKeeper(), serverName, regionInfo.getReplicaId(), state); } catch (KeeperException e) { throw new IOException(e); } }
private synchronized void init() throws IOException { refresh(); serverEventsListenerThread.start(); masterServices.getServerManager().registerListener(serverEventsListenerThread); failedOpenUpdaterThread = new FailedOpenUpdaterThread(masterServices.getConfiguration()); failedOpenUpdaterThread.start(); masterServices.getServerManager().registerListener(failedOpenUpdaterThread); }
@Override public void initialize(MasterServices master, MetricsMaster metricsMaster) throws IOException, UnsupportedOperationException { this.master = master; this.done = false; // setup the default procedure coordinator String name = master.getServerName().toString(); // get the configuration for the coordinator Configuration conf = master.getConfiguration(); long wakeFrequency = conf.getInt(BACKUP_WAKE_MILLIS_KEY, BACKUP_WAKE_MILLIS_DEFAULT); long timeoutMillis = conf.getLong(BACKUP_TIMEOUT_MILLIS_KEY,BACKUP_TIMEOUT_MILLIS_DEFAULT); int opThreads = conf.getInt(BACKUP_POOL_THREAD_NUMBER_KEY, BACKUP_POOL_THREAD_NUMBER_DEFAULT); // setup the default procedure coordinator ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads); ProcedureCoordinationManager coordManager = new ZKProcedureCoordinationManager(master); ProcedureCoordinatorRpcs comms = coordManager.getProcedureCoordinatorRpcs(getProcedureSignature(), name); this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency); }
@Override public void start(CoprocessorEnvironment env) throws IOException { if (!(env instanceof HasMasterServices)) { throw new IOException("Does not implement HMasterServices"); } master = ((HasMasterServices)env).getMasterServices(); groupInfoManager = RSGroupInfoManagerImpl.getInstance(master); groupAdminServer = new RSGroupAdminServer(master, groupInfoManager); Class<?> clazz = master.getConfiguration().getClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, null); if (!RSGroupableBalancer.class.isAssignableFrom(clazz)) { throw new IOException("Configured balancer does not support RegionServer groups."); } ZKWatcher zk = ((HasMasterServices)env).getMasterServices().getZooKeeper(); accessChecker = new AccessChecker(env.getConfiguration(), zk); // set the user-provider. this.userProvider = UserProvider.instantiate(env.getConfiguration()); }
@Override public void onConfigurationChange(Configuration conf) { master.getMasterProcedureExecutor().refreshConfiguration(conf); } }