@Inject public LogSaverStatusService(CConfiguration cConf, DiscoveryService discoveryService, @Named(Constants.LogSaver.LOG_SAVER_STATUS_HANDLER) Set<HttpHandler> handlers, MetricsCollectionService metricsCollectionService) { this.discoveryService = discoveryService; this.httpService = NettyHttpService.builder(LogSaverStatusService.class.getName()) .setHttpHandlers(handlers) .setHandlerHooks(ImmutableList.of(new MetricsReporterHook(metricsCollectionService, Constants.LogSaver.LOG_SAVER_STATUS_HANDLER))) .setHost(cConf.get(Constants.LogSaver.ADDRESS)) .build(); }
public SparkExecutionService(LocationFactory locationFactory, String host, ProgramRunId programRunId, @Nullable WorkflowToken workflowToken) { this.locationFactory = locationFactory; this.httpServer = NettyHttpService.builder(programRunId.getProgram() + "-spark-exec-service") .setHttpHandlers(Collections.singletonList(new SparkControllerHandler())) .setHost(host) .setExceptionHandler(new HttpExceptionHandler()) .build(); this.stopping = new AtomicBoolean(); this.stopLatch = new CountDownLatch(1); this.programRunId = programRunId; this.workflowToken = workflowToken; }
public SparkExecutionService(LocationFactory locationFactory, String host, ProgramRunId programRunId, @Nullable WorkflowToken workflowToken) { this.locationFactory = locationFactory; this.httpServer = NettyHttpService.builder(programRunId.getProgram() + "-spark-exec-service") .setHttpHandlers(Collections.singletonList(new SparkControllerHandler())) .setHost(host) .setExceptionHandler(new HttpExceptionHandler()) .build(); this.stopping = new AtomicBoolean(); this.stopLatch = new CountDownLatch(1); this.programRunId = programRunId; this.workflowToken = workflowToken; }
public MockProvisionerService(String id, String serverUrl, int totalCapacity, long taskMs, long msBetweenTasks, int failureRate) { this.id = id; this.totalCapacity = totalCapacity; NettyHttpService.Builder builder = NettyHttpService.builder(); builder.addHttpHandlers(Lists.newArrayList(new MockProvisionerHandler())); builder.setHost("localhost"); builder.setPort(0); builder.setConnectionBacklog(20000); builder.setExecThreadPoolSize(5); builder.setBossThreadPoolSize(1); builder.setWorkerThreadPoolSize(10); this.httpService = builder.build(); this.httpClient = HttpClients.createDefault(); this.heartbeatRequest = new HttpPost(serverUrl + "/provisioners/" + id + "/heartbeat"); this.registerRequest = new HttpPut(serverUrl + "/provisioners/" + id); this.deregisterRequest = new HttpDelete(serverUrl + "/provisioners/" + id); this.workerService = new MockProvisionerWorkerService(id, serverUrl, totalCapacity, taskMs, msBetweenTasks, failureRate); }
@Override protected void startUp() throws Exception { LOG.info("Starting Metadata Service"); httpService = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.METADATA_SERVICE) .setHttpHandlers(handlers) .setExceptionHandler(new HttpExceptionHandler()) .setHandlerHooks(ImmutableList.of(new MetricsReporterHook(metricsCollectionService, Constants.Service.METADATA_SERVICE))) .setHost(cConf.get(Constants.Metadata.SERVICE_BIND_ADDRESS)) .setPort(cConf.getInt(Constants.Metadata.SERVICE_BIND_PORT)) .setWorkerThreadPoolSize(cConf.getInt(Constants.Metadata.SERVICE_WORKER_THREADS)) .setExecThreadPoolSize(cConf.getInt(Constants.Metadata.SERVICE_EXEC_THREADS)) .setConnectionBacklog(20000) .build(); httpService.start(); InetSocketAddress socketAddress = httpService.getBindAddress(); LOG.info("Metadata service running at {}", socketAddress); cancelDiscovery = discoveryService.register( ResolvingDiscoverable.of(new Discoverable(Constants.Service.METADATA_SERVICE, socketAddress))); }
@Inject public ExploreExecutorService(CConfiguration cConf, DiscoveryService discoveryService, MetricsCollectionService metricsCollectionService, ExploreService exploreService, @Named(Constants.Service.EXPLORE_HTTP_USER_SERVICE) Set<HttpHandler> handlers) { this.exploreService = exploreService; this.discoveryService = discoveryService; this.startOnDemand = cConf.getBoolean(Constants.Explore.START_ON_DEMAND); int workerThreads = cConf.getInt(Constants.Explore.WORKER_THREADS, 10); int execThreads = cConf.getInt(Constants.Explore.EXEC_THREADS, 10); this.httpService = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.EXPLORE_HTTP_USER_SERVICE) .setHttpHandlers(handlers) .setHost(cConf.get(Constants.Explore.SERVER_ADDRESS)) .setPort(cConf.getInt(Constants.Explore.SERVER_PORT)) .setHandlerHooks(ImmutableList.of( new MetricsReporterHook(metricsCollectionService, Constants.Service.EXPLORE_HTTP_USER_SERVICE))) .setWorkerThreadPoolSize(workerThreads) .setExecThreadPoolSize(execThreads) .setConnectionBacklog(cConf.getInt(Constants.Explore.BACKLOG_CONNECTIONS, 20000)) .build(); }
@Inject public DatasetOpExecutorService(CConfiguration cConf, DiscoveryService discoveryService, MetricsCollectionService metricsCollectionService, @Named(Constants.Service.DATASET_EXECUTOR) Set<HttpHandler> handlers) { this.discoveryService = discoveryService; int workerThreads = cConf.getInt(Constants.Dataset.Executor.WORKER_THREADS, 10); int execThreads = cConf.getInt(Constants.Dataset.Executor.EXEC_THREADS, 30); this.httpService = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.DATASET_EXECUTOR) .setHttpHandlers(handlers) .setHost(cConf.get(Constants.Dataset.Executor.ADDRESS)) .setPort(cConf.getInt(Constants.Dataset.Executor.PORT)) .setHandlerHooks(ImmutableList.of( new MetricsReporterHook(metricsCollectionService, Constants.Service.DATASET_EXECUTOR))) .setWorkerThreadPoolSize(workerThreads) .setExecThreadPoolSize(execThreads) .setConnectionBacklog(20000) .build(); }
public StubProvisioner() { NettyHttpService.Builder builder = NettyHttpService.builder(); builder.addHttpHandlers(ImmutableSet.of(new StubTenantHandler())); builder.setHost("localhost"); builder.setPort(0); builder.setConnectionBacklog(20000); builder.setExecThreadPoolSize(1); builder.setBossThreadPoolSize(1); builder.setWorkerThreadPoolSize(1); this.httpService = builder.build(); this.tenantPutCounts = HashMultiset.create(); this.tenantResourcePutCounts = HashMultiset.create(); this.tenantWorkerPutCounts = HashMultiset.create(); this.tenantDeleteCounts = HashMultiset.create(); }
@Inject public StreamHttpService(CConfiguration cConf, DiscoveryService discoveryService, @Named(Constants.Stream.STREAM_HANDLER) Set<HttpHandler> handlers, @Nullable MetricsCollectionService metricsCollectionService) { this.discoveryService = discoveryService; int workerThreads = cConf.getInt(Constants.Stream.WORKER_THREADS, 10); this.httpService = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.STREAMS) .setHttpHandlers(handlers) .setHandlerHooks(ImmutableList.of(new MetricsReporterHook(metricsCollectionService, Constants.Stream.STREAM_HANDLER))) .setHost(cConf.get(Constants.Stream.ADDRESS)) .setPort(cConf.getInt(Constants.Stream.PORT)) .setWorkerThreadPoolSize(workerThreads) .setExecThreadPoolSize(0) // Execution happens in the io worker thread directly .setConnectionBacklog(20000) .build(); }
@Inject public DatasetOpExecutorService(CConfiguration cConf, DiscoveryService discoveryService, MetricsCollectionService metricsCollectionService, @Named(Constants.Service.DATASET_EXECUTOR) Set<HttpHandler> handlers) { this.discoveryService = discoveryService; int workerThreads = cConf.getInt(Constants.Dataset.Executor.WORKER_THREADS, 10); int execThreads = cConf.getInt(Constants.Dataset.Executor.EXEC_THREADS, 30); this.httpService = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.DATASET_EXECUTOR) .setHttpHandlers(handlers) .setHost(cConf.get(Constants.Dataset.Executor.ADDRESS)) .setPort(cConf.getInt(Constants.Dataset.Executor.PORT)) .setHandlerHooks(ImmutableList.of( new MetricsReporterHook(metricsCollectionService, Constants.Service.DATASET_EXECUTOR))) .setWorkerThreadPoolSize(workerThreads) .setExecThreadPoolSize(execThreads) .setConnectionBacklog(20000) .build(); }
@Inject public TransactionHttpService(CConfiguration cConf, @Named(Constants.Service.TRANSACTION_HTTP) Set<HttpHandler> handlers, DiscoveryService discoveryService, MetricsCollectionService metricsCollectionService) { // netty http server config String address = cConf.get(Constants.Transaction.Container.ADDRESS); NettyHttpService.Builder builder = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.TRANSACTION_HTTP); builder.setHandlerHooks(ImmutableList.of(new MetricsReporterHook(metricsCollectionService, Constants.Service.TRANSACTION_HTTP))); builder.setHttpHandlers(handlers); builder.setHost(address); this.httpService = builder.build(); this.discoveryService = discoveryService; }
public SparkExecutionService(LocationFactory locationFactory, String host, ProgramRunId programRunId, @Nullable WorkflowToken workflowToken) { this.locationFactory = locationFactory; this.httpServer = NettyHttpService.builder(programRunId.getProgram() + "-spark-exec-service") .setHttpHandlers(Collections.singletonList(new SparkControllerHandler())) .setHost(host) .setExceptionHandler(new HttpExceptionHandler()) .build(); this.stopping = new AtomicBoolean(); this.stopLatch = new CountDownLatch(1); this.programRunId = programRunId; this.workflowToken = workflowToken; }
@Inject public MetricsProcessorStatusService (CConfiguration cConf, DiscoveryService discoveryService, @Named(Constants.MetricsProcessor.METRICS_PROCESSOR_STATUS_HANDLER) Set<HttpHandler> handlers, MetricsCollectionService metricsCollectionService) { this.discoveryService = discoveryService; this.httpService = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.METRICS_PROCESSOR) .setHttpHandlers(handlers) .setHandlerHooks(ImmutableList.of(new MetricsReporterHook(metricsCollectionService, Constants.Service.METRICS_PROCESSOR))) .setHost(cConf.get(Constants.MetricsProcessor.ADDRESS)) .build(); }
@Inject public MetricsProcessorStatusService (CConfiguration cConf, DiscoveryService discoveryService, @Named(Constants.MetricsProcessor.METRICS_PROCESSOR_STATUS_HANDLER) Set<HttpHandler> handlers, MetricsCollectionService metricsCollectionService) { this.discoveryService = discoveryService; this.httpService = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.METRICS_PROCESSOR) .setHttpHandlers(handlers) .setHandlerHooks(ImmutableList.of(new MetricsReporterHook(metricsCollectionService, Constants.MetricsProcessor.METRICS_PROCESSOR_STATUS_HANDLER))) .setHost(cConf.get(Constants.MetricsProcessor.ADDRESS)) .build(); }
SparkDriverHttpService(String programName, String hostname, HttpHandler...handlers) { this.httpServer = NettyHttpService.builder(programName + "-http-service") .setHttpHandlers(handlers) .setExceptionHandler(new HttpExceptionHandler()) .setHost(hostname) .build(); }
@Inject public LogSaverStatusService(CConfiguration cConf, DiscoveryService discoveryService, @Named(Constants.LogSaver.LOG_SAVER_STATUS_HANDLER) Set<HttpHandler> handlers, MetricsCollectionService metricsCollectionService) { this.discoveryService = discoveryService; this.httpService = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.LOGSAVER) .setHttpHandlers(handlers) .setHandlerHooks(ImmutableList.of(new MetricsReporterHook(metricsCollectionService, Constants.Service.LOGSAVER))) .setHost(cConf.get(Constants.LogSaver.ADDRESS)) .build(); }
SparkDriverHttpService(String programName, String hostname, HttpHandler...handlers) { this.httpServer = NettyHttpService.builder(programName + "-http-service") .setHttpHandlers(handlers) .setExceptionHandler(new HttpExceptionHandler()) .setHost(hostname) .build(); }
SparkDriverHttpService(String programName, String hostname, HttpHandler...handlers) { this.httpServer = NettyHttpService.builder(programName + "-http-service") .setHttpHandlers(handlers) .setExceptionHandler(new HttpExceptionHandler()) .setHost(hostname) .build(); }
@Inject LogQueryService(CConfiguration cConf, @Named(Constants.Service.LOG_QUERY) Set<HttpHandler> handlers, DiscoveryService discoveryService, MetricsCollectionService metricsCollectionService) { this.discoveryService = discoveryService; this.httpServer = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.LOG_QUERY) .setHttpHandlers(handlers) .setHandlerHooks(ImmutableList.of(new MetricsReporterHook(metricsCollectionService, Constants.Service.LOG_QUERY))) .setHost(cConf.get(Constants.LogQuery.ADDRESS)) .setPort(cConf.getInt(Constants.LogQuery.PORT)) .build(); }
public DummyService(int port, HttpHandler handler) { NettyHttpService.Builder builder = NettyHttpService.builder(); builder.addHttpHandlers(ImmutableSet.of(handler)); builder.setHost("localhost"); builder.setPort(port); builder.setConnectionBacklog(20000); builder.setExecThreadPoolSize(10); builder.setBossThreadPoolSize(1); builder.setWorkerThreadPoolSize(1); this.httpService = builder.build(); }