public SparkExecutionService(LocationFactory locationFactory, String host, ProgramRunId programRunId, @Nullable WorkflowToken workflowToken) { this.locationFactory = locationFactory; this.httpServer = NettyHttpService.builder(programRunId.getProgram() + "-spark-exec-service") .setHttpHandlers(Collections.singletonList(new SparkControllerHandler())) .setHost(host) .setExceptionHandler(new HttpExceptionHandler()) .build(); this.stopping = new AtomicBoolean(); this.stopLatch = new CountDownLatch(1); this.programRunId = programRunId; this.workflowToken = workflowToken; }
public SparkExecutionService(LocationFactory locationFactory, String host, ProgramRunId programRunId, @Nullable WorkflowToken workflowToken) { this.locationFactory = locationFactory; this.httpServer = NettyHttpService.builder(programRunId.getProgram() + "-spark-exec-service") .setHttpHandlers(Collections.singletonList(new SparkControllerHandler())) .setHost(host) .setExceptionHandler(new HttpExceptionHandler()) .build(); this.stopping = new AtomicBoolean(); this.stopLatch = new CountDownLatch(1); this.programRunId = programRunId; this.workflowToken = workflowToken; }
public SparkExecutionService(LocationFactory locationFactory, String host, ProgramRunId programRunId, @Nullable WorkflowToken workflowToken) { this.locationFactory = locationFactory; this.httpServer = NettyHttpService.builder(programRunId.getProgram() + "-spark-exec-service") .setHttpHandlers(Collections.singletonList(new SparkControllerHandler())) .setHost(host) .setExceptionHandler(new HttpExceptionHandler()) .build(); this.stopping = new AtomicBoolean(); this.stopLatch = new CountDownLatch(1); this.programRunId = programRunId; this.workflowToken = workflowToken; }
@Override protected void startUp() throws Exception { LoggingContextAccessor.setLoggingContext(new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.RUNTIME_HTTP)); // Enable SSL for communication. NettyHttpService.Builder builder = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.RUNTIME_HTTP) .setHttpHandlers(new RuntimeHandler(cConf, messagingContext)) .setExceptionHandler(new HttpExceptionHandler()) .setHost(InetAddress.getLoopbackAddress().getHostName()); httpService = new HttpsEnabler() .setKeyStore(keyStore, ""::toCharArray) .setTrustStore(trustStore) .enable(builder) .build(); httpService.start(); // Writes the port to a local file Retries.runWithRetries( () -> { String content = GSON.toJson(new RuntimeMonitorServerInfo(httpService.getBindAddress().getPort())); java.nio.file.Path infoFile = Paths.get(cConf.get(Constants.RuntimeMonitor.SERVER_INFO_FILE)); Files.deleteIfExists(infoFile); Files.move(Files.write(Files.createTempFile(infoFile.getFileName().toString(), ".tmp"), Collections.singletonList(content)), infoFile); }, RetryStrategies.fixDelay(1, TimeUnit.SECONDS), IOException.class::isInstance); LOG.info("Runtime monitor server started on {}", httpService.getBindAddress()); }
.setExecThreadPoolSize(cConf.getInt(Constants.MessagingSystem.HTTP_SERVER_EXECUTOR_THREADS)) .setHttpChunkLimit(cConf.getInt(Constants.MessagingSystem.HTTP_SERVER_MAX_REQUEST_SIZE_MB) * 1024 * 1024) .setExceptionHandler(new HttpExceptionHandler() { @Override public void handle(Throwable t, HttpRequest request, HttpResponder responder) {
@Override protected void startUp() throws Exception { LoggingContextAccessor.setLoggingContext(new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.RUNTIME_HTTP)); // Enable SSL for communication. NettyHttpService.Builder builder = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.RUNTIME_HTTP) .setHttpHandlers(new RuntimeHandler(cConf, messagingContext)) .setExceptionHandler(new HttpExceptionHandler()) .setHost(InetAddress.getLoopbackAddress().getHostName()); httpService = new HttpsEnabler() .setKeyStore(keyStore, ""::toCharArray) .setTrustStore(trustStore) .enable(builder) .build(); httpService.start(); // Writes the port to a local file Retries.runWithRetries( () -> { String content = GSON.toJson(new RuntimeMonitorServerInfo(httpService.getBindAddress().getPort())); java.nio.file.Path infoFile = Paths.get(cConf.get(Constants.RuntimeMonitor.SERVER_INFO_FILE)); Files.deleteIfExists(infoFile); Files.move(Files.write(Files.createTempFile(infoFile.getFileName().toString(), ".tmp"), Collections.singletonList(content)), infoFile); }, RetryStrategies.fixDelay(1, TimeUnit.SECONDS), IOException.class::isInstance); LOG.info("Runtime monitor server started on {}", httpService.getBindAddress()); }
.setExecThreadPoolSize(cConf.getInt(Constants.MessagingSystem.HTTP_SERVER_EXECUTOR_THREADS)) .setHttpChunkLimit(cConf.getInt(Constants.MessagingSystem.HTTP_SERVER_MAX_REQUEST_SIZE_MB) * 1024 * 1024) .setExceptionHandler(new HttpExceptionHandler() { @Override public void handle(Throwable t, HttpRequest request, HttpResponder responder) {
@Override protected void startUp() throws Exception { LOG.info("Starting Metadata Service"); httpService = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.METADATA_SERVICE) .setHttpHandlers(handlers) .setExceptionHandler(new HttpExceptionHandler()) .setHandlerHooks(ImmutableList.of(new MetricsReporterHook(metricsCollectionService, Constants.Service.METADATA_SERVICE))) .setHost(cConf.get(Constants.Metadata.SERVICE_BIND_ADDRESS)) .setPort(cConf.getInt(Constants.Metadata.SERVICE_BIND_PORT)) .setWorkerThreadPoolSize(cConf.getInt(Constants.Metadata.SERVICE_WORKER_THREADS)) .setExecThreadPoolSize(cConf.getInt(Constants.Metadata.SERVICE_EXEC_THREADS)) .setConnectionBacklog(20000) .build(); httpService.start(); // Only first instance will run the migration thread. if (Boolean.valueOf(cConf.get(Constants.Dataset.Executor.IS_UPGRADE_NEEDED, "false"))) { metadataMigrator.start(); } InetSocketAddress socketAddress = httpService.getBindAddress(); LOG.info("Metadata service running at {}", socketAddress); cancelDiscovery = discoveryService.register( ResolvingDiscoverable.of(new Discoverable(Constants.Service.METADATA_SERVICE, socketAddress))); }
@BeforeClass public static void setUp() throws Exception { CConfiguration conf = CConfiguration.create(); conf.set(Constants.Security.Store.FILE_PATH, TEMP_FOLDER.newFolder().getAbsolutePath()); SConfiguration sConf = SConfiguration.create(); sConf.set(Constants.Security.Store.FILE_PASSWORD, "secret"); InMemoryNamespaceAdmin namespaceClient = new InMemoryNamespaceAdmin(); NamespaceMeta namespaceMeta = new NamespaceMeta.Builder() .setName(NAMESPACE1) .build(); namespaceClient.create(namespaceMeta); FileSecureStoreService fileSecureStoreService = new FileSecureStoreService(conf, sConf, namespaceClient); // Starts a mock server to handle remote secure store requests httpService = NettyHttpService.builder("remoteSecureStoreTest") .setHttpHandlers(new SecureStoreHandler(fileSecureStoreService, fileSecureStoreService)) .setExceptionHandler(new HttpExceptionHandler()) .build(); httpService.start(); InMemoryDiscoveryService discoveryService = new InMemoryDiscoveryService(); discoveryService.register(new Discoverable(Constants.Service.SECURE_STORE_SERVICE, httpService.getBindAddress())); remoteSecureStore = new RemoteSecureStore(discoveryService); }
@Override protected void startUp() throws Exception { LOG.info("Starting Metadata Service"); httpService = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.METADATA_SERVICE) .setHttpHandlers(handlers) .setExceptionHandler(new HttpExceptionHandler()) .setHandlerHooks(ImmutableList.of(new MetricsReporterHook(metricsCollectionService, Constants.Service.METADATA_SERVICE))) .setHost(cConf.get(Constants.Metadata.SERVICE_BIND_ADDRESS)) .setPort(cConf.getInt(Constants.Metadata.SERVICE_BIND_PORT)) .setWorkerThreadPoolSize(cConf.getInt(Constants.Metadata.SERVICE_WORKER_THREADS)) .setExecThreadPoolSize(cConf.getInt(Constants.Metadata.SERVICE_EXEC_THREADS)) .setConnectionBacklog(20000) .build(); httpService.start(); InetSocketAddress socketAddress = httpService.getBindAddress(); LOG.info("Metadata service running at {}", socketAddress); cancelDiscovery = discoveryService.register( ResolvingDiscoverable.of(new Discoverable(Constants.Service.METADATA_SERVICE, socketAddress))); }
SparkDriverHttpService(String programName, String hostname, HttpHandler...handlers) { this.httpServer = NettyHttpService.builder(programName + "-http-service") .setHttpHandlers(handlers) .setExceptionHandler(new HttpExceptionHandler()) .setHost(hostname) .build(); }
SparkDriverHttpService(String programName, String hostname, HttpHandler...handlers) { this.httpServer = NettyHttpService.builder(programName + "-http-service") .setHttpHandlers(handlers) .setExceptionHandler(new HttpExceptionHandler()) .setHost(hostname) .build(); }
SparkDriverHttpService(String programName, String hostname, HttpHandler...handlers) { this.httpServer = NettyHttpService.builder(programName + "-http-service") .setHttpHandlers(handlers) .setExceptionHandler(new HttpExceptionHandler()) .setHost(hostname) .build(); }