public MockProvisionerService(String id, String serverUrl, int totalCapacity, long taskMs, long msBetweenTasks, int failureRate) { this.id = id; this.totalCapacity = totalCapacity; NettyHttpService.Builder builder = NettyHttpService.builder(); builder.addHttpHandlers(Lists.newArrayList(new MockProvisionerHandler())); builder.setHost("localhost"); builder.setPort(0); builder.setConnectionBacklog(20000); builder.setExecThreadPoolSize(5); builder.setBossThreadPoolSize(1); builder.setWorkerThreadPoolSize(10); this.httpService = builder.build(); this.httpClient = HttpClients.createDefault(); this.heartbeatRequest = new HttpPost(serverUrl + "/provisioners/" + id + "/heartbeat"); this.registerRequest = new HttpPut(serverUrl + "/provisioners/" + id); this.deregisterRequest = new HttpDelete(serverUrl + "/provisioners/" + id); this.workerService = new MockProvisionerWorkerService(id, serverUrl, totalCapacity, taskMs, msBetweenTasks, failureRate); }
@Inject public StreamHttpService(CConfiguration cConf, DiscoveryService discoveryService, @Named(Constants.Stream.STREAM_HANDLER) Set<HttpHandler> handlers, @Nullable MetricsCollectionService metricsCollectionService) { this.discoveryService = discoveryService; int workerThreads = cConf.getInt(Constants.Stream.WORKER_THREADS, 10); this.httpService = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.STREAMS) .setHttpHandlers(handlers) .setHandlerHooks(ImmutableList.of(new MetricsReporterHook(metricsCollectionService, Constants.Stream.STREAM_HANDLER))) .setHost(cConf.get(Constants.Stream.ADDRESS)) .setPort(cConf.getInt(Constants.Stream.PORT)) .setWorkerThreadPoolSize(workerThreads) .setExecThreadPoolSize(0) // Execution happens in the io worker thread directly .setConnectionBacklog(20000) .build(); }
@Inject public MetricsQueryService(CConfiguration cConf, @Named(Constants.Service.METRICS) Set<HttpHandler> handlers, DiscoveryService discoveryService, @Nullable MetricsCollectionService metricsCollectionService) { // netty http server config String address = cConf.get(Constants.Metrics.ADDRESS); int backlogcnxs = cConf.getInt(Constants.Metrics.BACKLOG_CONNECTIONS, 20000); int execthreads = cConf.getInt(Constants.Metrics.EXEC_THREADS, 20); int bossthreads = cConf.getInt(Constants.Metrics.BOSS_THREADS, 1); int workerthreads = cConf.getInt(Constants.Metrics.WORKER_THREADS, 10); NettyHttpService.Builder builder = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.METRICS); builder.setHttpHandlers(handlers); builder.setHandlerHooks(ImmutableList.of(new MetricsReporterHook(metricsCollectionService, Constants.Service.METRICS))); builder.setHost(address); builder.setConnectionBacklog(backlogcnxs); builder.setExecThreadPoolSize(execthreads); builder.setBossThreadPoolSize(bossthreads); builder.setWorkerThreadPoolSize(workerthreads); this.httpService = builder.build(); this.discoveryService = discoveryService; LOG.info("Configuring MetricsService " + ", address: " + address + ", backlog connections: " + backlogcnxs + ", execthreads: " + execthreads + ", bossthreads: " + bossthreads + ", workerthreads: " + workerthreads); }
.setBossThreadPoolSize(cConf.getInt(Constants.AppFabric.BOSS_THREADS, Constants.AppFabric.DEFAULT_BOSS_THREADS)) .setWorkerThreadPoolSize(cConf.getInt(Constants.AppFabric.WORKER_THREADS, Constants.AppFabric.DEFAULT_WORKER_THREADS)); if (sslEnabled) {
.setBossThreadPoolSize(cConf.getInt(Constants.AppFabric.BOSS_THREADS, Constants.AppFabric.DEFAULT_BOSS_THREADS)) .setWorkerThreadPoolSize(cConf.getInt(Constants.AppFabric.WORKER_THREADS, Constants.AppFabric.DEFAULT_WORKER_THREADS)); if (sslEnabled) {
.setExecThreadPoolSize(cConf.getInt(Constants.Dataset.Manager.EXEC_THREADS)) .setBossThreadPoolSize(cConf.getInt(Constants.Dataset.Manager.BOSS_THREADS)) .setWorkerThreadPoolSize(cConf.getInt(Constants.Dataset.Manager.WORKER_THREADS)) .build(); this.discoveryService = discoveryService;
.setExecThreadPoolSize(cConf.getInt(Constants.Dataset.Manager.EXEC_THREADS)) .setBossThreadPoolSize(cConf.getInt(Constants.Dataset.Manager.BOSS_THREADS)) .setWorkerThreadPoolSize(cConf.getInt(Constants.Dataset.Manager.WORKER_THREADS)) .build(); this.discoveryService = discoveryService;
builder.setExecThreadPoolSize(numExecThreads); builder.setBossThreadPoolSize(1); builder.setWorkerThreadPoolSize(numWorkerThreads); if (securityEnabled) { builder.modifyChannelPipeline(new Function<ChannelPipeline, ChannelPipeline>() {
.setHandlerHooks(ImmutableList.of( new MetricsReporterHook(metricsCollectionService, Constants.Service.MESSAGING_SERVICE))) .setWorkerThreadPoolSize(cConf.getInt(Constants.MessagingSystem.HTTP_SERVER_WORKER_THREADS)) .setExecThreadPoolSize(cConf.getInt(Constants.MessagingSystem.HTTP_SERVER_EXECUTOR_THREADS)) .setHttpChunkLimit(cConf.getInt(Constants.MessagingSystem.HTTP_SERVER_MAX_REQUEST_SIZE_MB) * 1024 * 1024)
.setHandlerHooks(ImmutableList.of( new MetricsReporterHook(metricsCollectionService, Constants.Service.MESSAGING_SERVICE))) .setWorkerThreadPoolSize(cConf.getInt(Constants.MessagingSystem.HTTP_SERVER_WORKER_THREADS)) .setExecThreadPoolSize(cConf.getInt(Constants.MessagingSystem.HTTP_SERVER_EXECUTOR_THREADS)) .setHttpChunkLimit(cConf.getInt(Constants.MessagingSystem.HTTP_SERVER_MAX_REQUEST_SIZE_MB) * 1024 * 1024)
@Inject public MetricsQueryService(CConfiguration cConf, @Named(Constants.Service.METRICS) Set<HttpHandler> handlers, DiscoveryService discoveryService, @Nullable MetricsCollectionService metricsCollectionService) { // netty http server config String address = cConf.get(Constants.Metrics.ADDRESS); int backlogcnxs = cConf.getInt(Constants.Metrics.BACKLOG_CONNECTIONS, 20000); int execthreads = cConf.getInt(Constants.Metrics.EXEC_THREADS, 20); int bossthreads = cConf.getInt(Constants.Metrics.BOSS_THREADS, 1); int workerthreads = cConf.getInt(Constants.Metrics.WORKER_THREADS, 10); NettyHttpService.Builder builder = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.METRICS); builder.setHttpHandlers(handlers); builder.setHandlerHooks(ImmutableList.of(new MetricsReporterHook(metricsCollectionService, Constants.Service.METRICS))); builder.setHost(address); builder.setConnectionBacklog(backlogcnxs); builder.setExecThreadPoolSize(execthreads); builder.setBossThreadPoolSize(bossthreads); builder.setWorkerThreadPoolSize(workerthreads); this.httpService = builder.build(); this.discoveryService = discoveryService; LOG.info("Configuring MetricsService " + ", address: " + address + ", backlog connections: " + backlogcnxs + ", execthreads: " + execthreads + ", bossthreads: " + bossthreads + ", workerthreads: " + workerthreads); }
@Override protected void startUp() throws Exception { LOG.info("Starting Metadata Service"); httpService = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.METADATA_SERVICE) .setHttpHandlers(handlers) .setExceptionHandler(new HttpExceptionHandler()) .setHandlerHooks(ImmutableList.of(new MetricsReporterHook(metricsCollectionService, Constants.Service.METADATA_SERVICE))) .setHost(cConf.get(Constants.Metadata.SERVICE_BIND_ADDRESS)) .setPort(cConf.getInt(Constants.Metadata.SERVICE_BIND_PORT)) .setWorkerThreadPoolSize(cConf.getInt(Constants.Metadata.SERVICE_WORKER_THREADS)) .setExecThreadPoolSize(cConf.getInt(Constants.Metadata.SERVICE_EXEC_THREADS)) .setConnectionBacklog(20000) .build(); httpService.start(); // Only first instance will run the migration thread. if (Boolean.valueOf(cConf.get(Constants.Dataset.Executor.IS_UPGRADE_NEEDED, "false"))) { metadataMigrator.start(); } InetSocketAddress socketAddress = httpService.getBindAddress(); LOG.info("Metadata service running at {}", socketAddress); cancelDiscovery = discoveryService.register( ResolvingDiscoverable.of(new Discoverable(Constants.Service.METADATA_SERVICE, socketAddress))); }
@Inject public ExploreExecutorService(CConfiguration cConf, DiscoveryService discoveryService, MetricsCollectionService metricsCollectionService, ExploreService exploreService, @Named(Constants.Service.EXPLORE_HTTP_USER_SERVICE) Set<HttpHandler> handlers) { this.exploreService = exploreService; this.discoveryService = discoveryService; this.startOnDemand = cConf.getBoolean(Constants.Explore.START_ON_DEMAND); int workerThreads = cConf.getInt(Constants.Explore.WORKER_THREADS, 10); int execThreads = cConf.getInt(Constants.Explore.EXEC_THREADS, 10); this.httpService = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.EXPLORE_HTTP_USER_SERVICE) .setHttpHandlers(handlers) .setHost(cConf.get(Constants.Explore.SERVER_ADDRESS)) .setPort(cConf.getInt(Constants.Explore.SERVER_PORT)) .setHandlerHooks(ImmutableList.of( new MetricsReporterHook(metricsCollectionService, Constants.Service.EXPLORE_HTTP_USER_SERVICE))) .setWorkerThreadPoolSize(workerThreads) .setExecThreadPoolSize(execThreads) .setConnectionBacklog(cConf.getInt(Constants.Explore.BACKLOG_CONNECTIONS, 20000)) .build(); }
@Inject public ExploreExecutorService(CConfiguration cConf, DiscoveryService discoveryService, MetricsCollectionService metricsCollectionService, ExploreService exploreService, @Named(Constants.Service.EXPLORE_HTTP_USER_SERVICE) Set<HttpHandler> handlers) { this.exploreService = exploreService; this.discoveryService = discoveryService; this.startOnDemand = cConf.getBoolean(Constants.Explore.START_ON_DEMAND); int workerThreads = cConf.getInt(Constants.Explore.WORKER_THREADS, 10); int execThreads = cConf.getInt(Constants.Explore.EXEC_THREADS, 10); this.httpService = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.EXPLORE_HTTP_USER_SERVICE) .setHttpHandlers(handlers) .setHost(cConf.get(Constants.Explore.SERVER_ADDRESS)) .setPort(cConf.getInt(Constants.Explore.SERVER_PORT)) .setHandlerHooks(ImmutableList.of( new MetricsReporterHook(metricsCollectionService, Constants.Service.EXPLORE_HTTP_USER_SERVICE))) .setWorkerThreadPoolSize(workerThreads) .setExecThreadPoolSize(execThreads) .setConnectionBacklog(cConf.getInt(Constants.Explore.BACKLOG_CONNECTIONS, 20000)) .build(); }
@Override protected void startUp() throws Exception { LOG.info("Starting Metadata Service"); httpService = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.METADATA_SERVICE) .setHttpHandlers(handlers) .setExceptionHandler(new HttpExceptionHandler()) .setHandlerHooks(ImmutableList.of(new MetricsReporterHook(metricsCollectionService, Constants.Service.METADATA_SERVICE))) .setHost(cConf.get(Constants.Metadata.SERVICE_BIND_ADDRESS)) .setPort(cConf.getInt(Constants.Metadata.SERVICE_BIND_PORT)) .setWorkerThreadPoolSize(cConf.getInt(Constants.Metadata.SERVICE_WORKER_THREADS)) .setExecThreadPoolSize(cConf.getInt(Constants.Metadata.SERVICE_EXEC_THREADS)) .setConnectionBacklog(20000) .build(); httpService.start(); InetSocketAddress socketAddress = httpService.getBindAddress(); LOG.info("Metadata service running at {}", socketAddress); cancelDiscovery = discoveryService.register( ResolvingDiscoverable.of(new Discoverable(Constants.Service.METADATA_SERVICE, socketAddress))); }
@Inject public DatasetOpExecutorService(CConfiguration cConf, DiscoveryService discoveryService, MetricsCollectionService metricsCollectionService, @Named(Constants.Service.DATASET_EXECUTOR) Set<HttpHandler> handlers) { this.discoveryService = discoveryService; int workerThreads = cConf.getInt(Constants.Dataset.Executor.WORKER_THREADS, 10); int execThreads = cConf.getInt(Constants.Dataset.Executor.EXEC_THREADS, 30); this.httpService = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.DATASET_EXECUTOR) .setHttpHandlers(handlers) .setHost(cConf.get(Constants.Dataset.Executor.ADDRESS)) .setPort(cConf.getInt(Constants.Dataset.Executor.PORT)) .setHandlerHooks(ImmutableList.of( new MetricsReporterHook(metricsCollectionService, Constants.Service.DATASET_EXECUTOR))) .setWorkerThreadPoolSize(workerThreads) .setExecThreadPoolSize(execThreads) .setConnectionBacklog(20000) .build(); }
@Inject public DatasetOpExecutorService(CConfiguration cConf, DiscoveryService discoveryService, MetricsCollectionService metricsCollectionService, @Named(Constants.Service.DATASET_EXECUTOR) Set<HttpHandler> handlers) { this.discoveryService = discoveryService; int workerThreads = cConf.getInt(Constants.Dataset.Executor.WORKER_THREADS, 10); int execThreads = cConf.getInt(Constants.Dataset.Executor.EXEC_THREADS, 30); this.httpService = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.DATASET_EXECUTOR) .setHttpHandlers(handlers) .setHost(cConf.get(Constants.Dataset.Executor.ADDRESS)) .setPort(cConf.getInt(Constants.Dataset.Executor.PORT)) .setHandlerHooks(ImmutableList.of( new MetricsReporterHook(metricsCollectionService, Constants.Service.DATASET_EXECUTOR))) .setWorkerThreadPoolSize(workerThreads) .setExecThreadPoolSize(execThreads) .setConnectionBacklog(20000) .build(); }
public StubProvisioner() { NettyHttpService.Builder builder = NettyHttpService.builder(); builder.addHttpHandlers(ImmutableSet.of(new StubTenantHandler())); builder.setHost("localhost"); builder.setPort(0); builder.setConnectionBacklog(20000); builder.setExecThreadPoolSize(1); builder.setBossThreadPoolSize(1); builder.setWorkerThreadPoolSize(1); this.httpService = builder.build(); this.tenantPutCounts = HashMultiset.create(); this.tenantResourcePutCounts = HashMultiset.create(); this.tenantWorkerPutCounts = HashMultiset.create(); this.tenantDeleteCounts = HashMultiset.create(); }
public DummyService(int port, HttpHandler handler) { NettyHttpService.Builder builder = NettyHttpService.builder(); builder.addHttpHandlers(ImmutableSet.of(handler)); builder.setHost("localhost"); builder.setPort(port); builder.setConnectionBacklog(20000); builder.setExecThreadPoolSize(10); builder.setBossThreadPoolSize(1); builder.setWorkerThreadPoolSize(1); this.httpService = builder.build(); }