public Configuration getRequestConfig() { return _serverConf.subset(PINOT_SERVER_REQUEST); }
public Configuration getQueryExecutorConfig() { return _serverConf.subset(PINOT_SERVER_QUERY); }
public Configuration getMetricsConfig() { return _serverConf.subset(PINOT_SERVER_METRICS); }
public Configuration getSchedulerConfig() { return _serverConf.subset(PINOT_QUERY_SCHEDULER_PREFIX); }
public Configuration getConfig(String component) { return _serverConf.subset(PINOT_ + component); }
public Configuration getInstanceDataManagerConfig() { return _serverConf.subset(PINOT_SERVER_INSTANCE); }
public static void init(Configuration uploaderConfig) { Configuration httpsConfig = uploaderConfig.subset(HTTPS_PROTOCOL); if (httpsConfig.getBoolean(CONFIG_OF_CONTROLLER_HTTPS_ENABLED, false)) { _sslContext = new ClientSSLContextGenerator(httpsConfig.subset(CommonConstants.PREFIX_OF_SSL_SUBSET)).generate(); _controllerHttpsPort = httpsConfig.getInt(CONFIG_OF_CONTROLLER_HTTPS_PORT); } }
public NettyServerConfig getNettyConfig() throws ConfigurationException { return new NettyServerConfig(_serverConf.subset(PINOT_SERVER_NETTY)); }
public SegmentFetcherAndLoader(@Nonnull Configuration config, @Nonnull InstanceDataManager instanceDataManager, @Nonnull ZkHelixPropertyStore<ZNRecord> propertyStore) throws Exception { _instanceDataManager = instanceDataManager; _propertyStore = propertyStore; Configuration pinotFSConfig = config.subset(CommonConstants.Server.PREFIX_OF_CONFIG_OF_PINOT_FS_FACTORY); Configuration segmentFetcherFactoryConfig = config.subset(CommonConstants.Server.PREFIX_OF_CONFIG_OF_SEGMENT_FETCHER_FACTORY); Configuration pinotCrypterConfig = config.subset(CommonConstants.Server.PREFIX_OF_CONFIG_OF_PINOT_CRYPTER); PinotFSFactory.init(pinotFSConfig); SegmentFetcherFactory.getInstance().init(segmentFetcherFactoryConfig); PinotCrypterFactory.init(pinotCrypterConfig); _crypterConfig = config.subset(CommonConstants.Server.PREFIX_OF_CONFIG_OF_PINOT_CRYPTER); }
/** * Gets the metadata map associated with the instance. The properties that * will be looked up for this will be <code>namespace + ".metadata"</code>. * * <p> * For instance, if the given namespace is <code>eureka.appinfo</code>, the * metadata keys are searched under the namespace * <code>eureka.appinfo.metadata</code>. * </p> */ @Override public Map<String, String> getMetadataMap() { String metadataNamespace = namespace + INSTANCE_METADATA_PREFIX + "."; Map<String, String> metadataMap = new LinkedHashMap<String, String>(); Configuration config = (Configuration) configInstance.getBackingConfigurationSource(); String subsetPrefix = metadataNamespace.charAt(metadataNamespace.length() - 1) == '.' ? metadataNamespace.substring(0, metadataNamespace.length() - 1) : metadataNamespace; for (Iterator<String> iter = config.subset(subsetPrefix).getKeys(); iter.hasNext(); ) { String key = iter.next(); String value = config.getString(subsetPrefix + "." + key); metadataMap.put(key, value); } return metadataMap; }
public SegmentPrunerConfig(Configuration segmentPrunerConfig) throws ConfigurationException { _segmentPrunerSetConfig = segmentPrunerConfig; checkRequiredKeys(); String[] serviceClasses = _segmentPrunerSetConfig.getStringArray(SEGMENT_PRUNER_CLASS); for (String serviceClass : serviceClasses) { _segmentPrunerClassNameList.add(serviceClass); _segmentPrunerConfigurationList.add(_segmentPrunerSetConfig.subset(serviceClass)); } }
/** * Load Config */ private void loadConfigs() { List<String> tables = getTableNames(); for (String s : tables) { PerTableRoutingConfig cfg = new PerTableRoutingConfig(_brokerRoutingConfig.subset(s)); _tableRoutingCfg.put(s, cfg); } }
public void init(Configuration cfg) { if (cfg.containsKey(ROUTING_MODE_KEY)) { _routingMode = RoutingMode.valueOf(cfg.getString(ROUTING_MODE_KEY)); } if ((_routingMode == RoutingMode.CONFIG)) { _cfgBasedRouting.init(cfg.subset(CFG_BASED_ROUTING)); } Configuration connPoolCfg = cfg.subset(CONNECTION_POOL_CONFIG); if (connPoolCfg != null) { _connPool.init(connPoolCfg); } }
public QueryExecutorConfig(Configuration config) throws ConfigurationException { _queryExecutorConfig = config; checkRequiredKeys(); _segmentPrunerConfig = new SegmentPrunerConfig(_queryExecutorConfig.subset(QUERY_PRUNER)); _queryPlannerConfig = new QueryPlannerConfig(_queryExecutorConfig.subset(QUERY_PLANNER)); _timeOutMs = _queryExecutorConfig.getLong(TIME_OUT, -1); }
public static void init(Configuration fsConfig) { // Get schemes and their respective classes Iterator<String> keys = fsConfig.subset(CLASS).getKeys(); if (!keys.hasNext()) { LOGGER.info("Did not find any fs classes in the configuration"); } while (keys.hasNext()) { String key = keys.next(); String fsClassName = (String) fsConfig.getProperty(CLASS + "." + key); LOGGER.info("Got scheme {}, classname {}, starting to initialize", key, fsClassName); try { PinotFS pinotFS = (PinotFS) Class.forName(fsClassName).newInstance(); pinotFS.init(fsConfig.subset(key)); LOGGER.info("Initializing PinotFS for scheme {}, classname {}", key, fsClassName); _fileSystemMap.put(key, pinotFS); } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { LOGGER.error("Could not instantiate file system for class {}", fsClassName, e); throw new RuntimeException(e); } } if (!_fileSystemMap.containsKey(DEFAULT_FS_SCHEME)) { LOGGER.info("LocalPinotFS not configured, adding as default"); _fileSystemMap.put(DEFAULT_FS_SCHEME, new LocalPinotFS()); } }
/** * Initiate the segment fetcher factory. This method should only be called once. * @param segmentFetcherClassConfig Segment fetcher factory config * */ public void init(Configuration segmentFetcherClassConfig) throws ClassNotFoundException, IllegalAccessException, InstantiationException { @SuppressWarnings("unchecked") List<String> protocols = segmentFetcherClassConfig.getList(PROTOCOLS_KEY, DEFAULT_PROTOCOLS); for (String protocol : protocols) { String fetcherClass = segmentFetcherClassConfig .getString(protocol + FETCHER_CLASS_KEY_SUFFIX, DEFAULT_FETCHER_CLASS_MAP.get(protocol)); Preconditions.checkNotNull(fetcherClass, "No fetcher class defined for protocol: " + protocol); LOGGER.info("Creating a new segment fetcher for protocol: {} with class: {}", protocol, fetcherClass); SegmentFetcher segmentFetcher = (SegmentFetcher) Class.forName(fetcherClass).newInstance(); LOGGER.info("Initializing segment fetcher for protocol: {}", protocol); Configuration segmentFetcherConfig = segmentFetcherClassConfig.subset(protocol); logFetcherInitConfig(segmentFetcher, protocol, segmentFetcherConfig); segmentFetcher.init(segmentFetcherConfig); _segmentFetcherMap.put(protocol, segmentFetcher); } }
@Override protected void initHttpClient(Configuration configs) { SSLContext sslContext = new ClientSSLContextGenerator(configs.subset(CommonConstants.PREFIX_OF_SSL_SUBSET)).generate(); _httpClient = new FileUploadDownloadClient(sslContext); }
public BrokerServerBuilder(Configuration config, RoutingTable routingTable, TimeBoundaryService timeBoundaryService, LiveInstancesChangeListenerImpl liveInstanceChangeListener, TableQueryQuotaManager tableQueryQuotaManager) { _state.set(State.INIT); _config = config; _delayedShutdownTimeMs = config.getLong(DELAY_SHUTDOWN_TIME_MS_CONFIG, DEFAULT_DELAY_SHUTDOWN_TIME_MS); _routingTable = routingTable; _timeBoundaryService = timeBoundaryService; _liveInstanceChangeListener = liveInstanceChangeListener; _tableQueryQuotaManager = tableQueryQuotaManager; _accessControlFactory = AccessControlFactory.loadFactory(_config.subset(ACCESS_CONTROL_PREFIX)); _metricsRegistry = new MetricsRegistry(); MetricsHelper.initializeMetrics(config.subset(METRICS_CONFIG_PREFIX)); MetricsHelper.registerMetricsRegistry(_metricsRegistry); _brokerMetrics = new BrokerMetrics(_metricsRegistry, !_config.getBoolean(TABLE_LEVEL_METRICS_CONFIG, true)); _brokerMetrics.initializeGlobalMeters(); _brokerRequestHandler = buildRequestHandler(); _brokerAdminApplication = new BrokerAdminApiApplication(this); }
@Test public void testMetricsHelperRegistration() { listenerOneOkay = false; listenerTwoOkay = false; Map<String, String> configKeys = new HashMap<String, String>(); configKeys.put("pinot.broker.metrics.metricsRegistryRegistrationListeners", ListenerOne.class.getName() + "," + ListenerTwo.class.getName()); Configuration configuration = new MapConfiguration(configKeys); MetricsRegistry registry = new MetricsRegistry(); // Initialize the MetricsHelper and create a new timer MetricsHelper.initializeMetrics(configuration.subset("pinot.broker.metrics")); MetricsHelper.registerMetricsRegistry(registry); MetricsHelper.newTimer(registry, new MetricName(MetricsHelperTest.class, "dummy"), TimeUnit.MILLISECONDS, TimeUnit.MILLISECONDS); // Check that the two listeners fired assertTrue(listenerOneOkay); assertTrue(listenerTwoOkay); } }
public ConnectionPoolBrokerRequestHandler(Configuration config, RoutingTable routingTable, TimeBoundaryService timeBoundaryService, AccessControlFactory accessControlFactory, TableQueryQuotaManager tableQueryQuotaManager, BrokerMetrics brokerMetrics, LiveInstancesChangeListenerImpl liveInstanceChangeListener, MetricsRegistry metricsRegistry) { super(config, routingTable, timeBoundaryService, accessControlFactory, tableQueryQuotaManager, brokerMetrics); _liveInstanceChangeListener = liveInstanceChangeListener; TransportClientConf transportClientConf = new TransportClientConf(); transportClientConf.init(_config.subset(TRANSPORT_CONFIG_PREFIX)); // Set up connection pool _eventLoopGroup = new NioEventLoopGroup(); // Some of the client metrics use histogram which is doing synchronous operation, which are fixed overhead per // request/response // TODO: measure the overhead of this NettyClientMetrics clientMetrics = new NettyClientMetrics(metricsRegistry, "client_"); PooledNettyClientResourceManager resourceManager = new PooledNettyClientResourceManager(_eventLoopGroup, new HashedWheelTimer(), clientMetrics); _poolTimeoutExecutor = new ScheduledThreadPoolExecutor(50); _requestSenderPool = Executors.newCachedThreadPool(); ConnectionPoolConfig connectionPoolConfig = transportClientConf.getConnPool(); _connPool = new KeyedPoolImpl<>(connectionPoolConfig.getMinConnectionsPerServer(), connectionPoolConfig.getMaxConnectionsPerServer(), connectionPoolConfig.getIdleTimeoutMs(), connectionPoolConfig.getMaxBacklogPerServer(), resourceManager, _poolTimeoutExecutor, _requestSenderPool, metricsRegistry); resourceManager.setPool(_connPool); _scatterGather = new ScatterGatherImpl(_connPool, _requestSenderPool); }