public boolean emitTableLevelMetrics() { return _serverConf.getBoolean(PINOT_SERVER_TABLE_LEVEL_METRICS, true); } }
@Override public boolean isEnableDefaultColumns() { return _instanceDataManagerConfiguration.getBoolean(ENABLE_DEFAULT_COLUMNS, false); }
@Override public boolean isEnableSplitCommit() { return _instanceDataManagerConfiguration.getBoolean(ENABLE_SPLIT_COMMIT, false); }
@Override public boolean isRealtimeOffHeapAllocation() { return _instanceDataManagerConfiguration.getBoolean(REALTIME_OFFHEAP_ALLOCATION, false); }
@Override public boolean isDirectRealtimeOffheapAllocation() { return _instanceDataManagerConfiguration.getBoolean(DIRECT_REALTIME_OFFHEAP_ALLOCATION, false); }
public ClientSSLContextGenerator(Configuration sslConfig) { if (sslConfig.getBoolean(CONFIG_OF_ENABLE_SERVER_VERIFICATION, true)) { _serverCACertFile = sslConfig.getString(CONFIG_OF_SERVER_CA_CERT); } else { _serverCACertFile = null; LOGGER.warn("Https Server CA file not configured.. All servers will be trusted!"); } _keyStoreFile = sslConfig.getString(CONFIG_OF_CLIENT_PKCS12_FILE); _keyStorePassword = sslConfig.getString(CONFIG_OF_CLIENT_PKCS12_PASSWORD); if ((_keyStorePassword == null && _keyStoreFile != null) || (_keyStorePassword != null && _keyStoreFile == null)) { throw new IllegalArgumentException("Invalid configuration of keystore file and passowrd"); } }
private void waitForAllSegmentsLoaded() { if (_helixServerConfig.getBoolean(CommonConstants.Server.CONFIG_OF_STARTER_ENABLE_SEGMENTS_LOADING_CHECK, CommonConstants.Server.DEFAULT_STARTER_ENABLE_SEGMENTS_LOADING_CHECK)) { long startTime = System.currentTimeMillis();
.getBoolean(RebalanceUserConfigConstants.DOWNTIME, RebalanceUserConfigConstants.DEFAULT_DOWNTIME)) { setTargetState(idealState, segmentId, targetMap); return;
public static void init(Configuration uploaderConfig) { Configuration httpsConfig = uploaderConfig.subset(HTTPS_PROTOCOL); if (httpsConfig.getBoolean(CONFIG_OF_CONTROLLER_HTTPS_ENABLED, false)) { _sslContext = new ClientSSLContextGenerator(httpsConfig.subset(CommonConstants.PREFIX_OF_SSL_SUBSET)).generate(); _controllerHttpsPort = httpsConfig.getInt(CONFIG_OF_CONTROLLER_HTTPS_PORT); } }
return (O)config.getString(key); } else if (datatype==Boolean.class) { return (O)new Boolean(config.getBoolean(key)); } else if (datatype.isEnum()) { Enum[] constants = (Enum[])datatype.getEnumConstants();
public void stop() { try { LOGGER.info("Closing PinotFS classes"); PinotFSFactory.shutdown(); } catch (IOException e) { LOGGER.warn("Caught exception closing PinotFS classes", e); } _adminApiApplication.stop(); setShuttingDownStatus(true); // Total waiting time should include max query time. final long endTime = _maxShutdownWaitTimeMs + System.currentTimeMillis(); if (_helixServerConfig.getBoolean(CommonConstants.Server.CONFIG_OF_ENABLE_SHUTDOWN_DELAY, true)) { Uninterruptibles.sleepUninterruptibly(_maxQueryTimeMs, TimeUnit.MILLISECONDS); } waitUntilNoIncomingQueries(System.currentTimeMillis(), endTime); _helixManager.disconnect(); _serverInstance.shutDown(); waitUntilNoOnlineResources(System.currentTimeMillis(), endTime); }
private AlarmService initAlarmService() { String emailPassword = config.getString("yugong.alarm.email.password"); if (StringUtils.isNotEmpty(emailPassword)) { MailAlarmService alarmService = new MailAlarmService(); alarmService.setEmailPassword(emailPassword); alarmService.setEmailHost(config.getString("yugong.alarm.email.host")); alarmService.setEmailUsername(config.getString("yugong.alarm.email.username")); alarmService.setStmpPort(config.getInt("yugong.alarm.email.stmp.port", 465)); alarmService.setSslSupport(config.getBoolean("yugong.alarm.email.ssl.support", true)); alarmService.start(); return alarmService; } else { return new LogAlarmService(); } }
return (O)config.getString(key); } else if (dataType==Boolean.class) { return (O) Boolean.valueOf(config.getBoolean(key)); } else if (dataType.isEnum()) { Enum[] constants = (Enum[])dataType.getEnumConstants();
private YuGongContext initGlobalContext() { YuGongContext context = new YuGongContext(); logger.info("check source database connection ..."); context.setSourceDs(initDataSource("source")); logger.info("check source database is ok"); // if (sourceDbType.isOracle() && runMode.isAll()) { // preCheckMlogGrant(context.getSourceDs()); // } logger.info("check target database connection ..."); context.setTargetDs(initDataSource("target")); logger.info("check target database is ok"); context.setSourceEncoding(config.getString("yugong.database.source.encode", "UTF-8")); context.setTargetEncoding(config.getString("yugong.database.target.encode", "UTF-8")); context.setBatchApply(config.getBoolean("yugong.table.batchApply", true)); context.setOnceCrawNum(config.getInt("yugong.table.onceCrawNum", 200)); context.setTpsLimit(config.getInt("yugong.table.tpsLimit", 2000)); context.setIgnoreSchema(config.getBoolean("yugong.table.ignoreSchema", false)); context.setSkipApplierException(config.getBoolean("yugong.table.skipApplierException", false)); context.setRunMode(runMode); context.setmViewLogType(config.getString("yugong.table.inc.mviewlogtype", "PK")); context.setTablepks(getTablePKs(config.getString("yugong.table.inc.tablepks"))); return context; }
private void validateIdealStateRealtime(IdealState rebalancedIdealState, int nSegmentsCompleted, int nSegmentsConsuming, int targetNumReplicas, List<String> instancesCompleted, List<String> instancesConsuming, Configuration rebalanceUserConfig) { Assert.assertEquals(rebalancedIdealState.getPartitionSet().size(), nSegmentsCompleted + nSegmentsConsuming); for (String segment : rebalancedIdealState.getPartitionSet()) { Map<String, String> instanceStateMap = rebalancedIdealState.getInstanceStateMap(segment); Assert.assertEquals(instanceStateMap.size(), targetNumReplicas); boolean rebalanceConsuming = rebalanceUserConfig.getBoolean(RebalanceUserConfigConstants.INCLUDE_CONSUMING); if (segment.contains("consuming")) { if (rebalanceConsuming) { Assert.assertTrue(instancesConsuming.containsAll(instanceStateMap.keySet())); } } else { Assert.assertTrue(instancesCompleted.containsAll(instanceStateMap.keySet())); } } }
.getBoolean(RebalanceUserConfigConstants.DRYRUN, RebalanceUserConfigConstants.DEFAULT_DRY_RUN); if (!dryRun) { LOGGER.info("Updating replica group partition assignment for table {}", tableNameWithType);
/** * Rebalances ideal state object without updating it * @param idealState * @param tableConfig * @param targetNumReplicas * @param rebalanceUserConfig * @param newPartitionAssignment */ private IdealState rebalanceIdealState(IdealState idealState, TableConfig tableConfig, int targetNumReplicas, Configuration rebalanceUserConfig, PartitionAssignment newPartitionAssignment) { // if realtime and includeConsuming, then rebalance consuming segments if (tableConfig.getTableType().equals(CommonConstants.Helix.TableType.REALTIME)) { boolean includeConsuming = rebalanceUserConfig.getBoolean(RebalanceUserConfigConstants.INCLUDE_CONSUMING, DEFAULT_INCLUDE_CONSUMING); if (includeConsuming) { rebalanceConsumingSegments(idealState, newPartitionAssignment); } } // always rebalance serving segments rebalanceServingSegments(idealState, tableConfig, targetNumReplicas); idealState.setReplicas(Integer.toString(targetNumReplicas)); return idealState; }
rebalanceUserConfig.getBoolean(RebalanceUserConfigConstants.INCLUDE_CONSUMING, DEFAULT_INCLUDE_CONSUMING); if (includeConsuming) {
public BrokerServerBuilder(Configuration config, RoutingTable routingTable, TimeBoundaryService timeBoundaryService, LiveInstancesChangeListenerImpl liveInstanceChangeListener, TableQueryQuotaManager tableQueryQuotaManager) { _state.set(State.INIT); _config = config; _delayedShutdownTimeMs = config.getLong(DELAY_SHUTDOWN_TIME_MS_CONFIG, DEFAULT_DELAY_SHUTDOWN_TIME_MS); _routingTable = routingTable; _timeBoundaryService = timeBoundaryService; _liveInstanceChangeListener = liveInstanceChangeListener; _tableQueryQuotaManager = tableQueryQuotaManager; _accessControlFactory = AccessControlFactory.loadFactory(_config.subset(ACCESS_CONTROL_PREFIX)); _metricsRegistry = new MetricsRegistry(); MetricsHelper.initializeMetrics(config.subset(METRICS_CONFIG_PREFIX)); MetricsHelper.registerMetricsRegistry(_metricsRegistry); _brokerMetrics = new BrokerMetrics(_metricsRegistry, !_config.getBoolean(TABLE_LEVEL_METRICS_CONFIG, true)); _brokerMetrics.initializeGlobalMeters(); _brokerRequestHandler = buildRequestHandler(); _brokerAdminApplication = new BrokerAdminApiApplication(this); }
private RecordApplier chooseApplier(TableHolder tableHolder, YuGongContext context, RunMode runMode) { boolean concurrent = config.getBoolean("yugong.applier.concurrent.enable", true); int threadSize = config.getInt("yugong.applier.concurrent.size", 5); int splitSize = context.getOnceCrawNum() / threadSize;