public TaskManagementExecutor() { taskManagementExecutor = newScheduledThreadPool(5, threadsNamed("task-management-%s")); taskManagementExecutorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) taskManagementExecutor); }
public ClusterSizeMonitor( InternalNodeManager nodeManager, boolean includeCoordinator, int initializationMinCount, Duration initializationMaxWait, int executionMinCount, Duration executionMaxWait) { this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); this.includeCoordinator = includeCoordinator; checkArgument(initializationMinCount >= 0, "initializationMinCount is negative"); this.initializationMinCount = initializationMinCount; this.initializationMaxWait = requireNonNull(initializationMaxWait, "initializationMaxWait is null"); checkArgument(executionMinCount >= 0, "executionMinCount is negative"); this.executionMinCount = executionMinCount; this.executionMaxWait = requireNonNull(executionMaxWait, "executionMaxWait is null"); this.executor = newSingleThreadScheduledExecutor(threadsNamed("node-monitor-%s")); }
@VisibleForTesting public TaskExecutor( int runnerThreads, int minDrivers, int guaranteedNumberOfDriversPerTask, int maximumNumberOfDriversPerTask, EmbedVersion embedVersion, MultilevelSplitQueue splitQueue, Ticker ticker) { checkArgument(runnerThreads > 0, "runnerThreads must be at least 1"); checkArgument(guaranteedNumberOfDriversPerTask > 0, "guaranteedNumberOfDriversPerTask must be at least 1"); checkArgument(maximumNumberOfDriversPerTask > 0, "maximumNumberOfDriversPerTask must be at least 1"); checkArgument(guaranteedNumberOfDriversPerTask <= maximumNumberOfDriversPerTask, "guaranteedNumberOfDriversPerTask cannot be greater than maximumNumberOfDriversPerTask"); // we manage thread pool size directly, so create an unlimited pool this.executor = newCachedThreadPool(threadsNamed("task-processor-%s")); this.executorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) executor); this.runnerThreads = runnerThreads; this.embedVersion = requireNonNull(embedVersion, "embedVersion is null"); this.ticker = requireNonNull(ticker, "ticker is null"); this.minimumNumberOfDrivers = minDrivers; this.guaranteedNumberOfDriversPerTask = guaranteedNumberOfDriversPerTask; this.maximumNumberOfDriversPerTask = maximumNumberOfDriversPerTask; this.waitingSplits = requireNonNull(splitQueue, "splitQueue is null"); this.tasks = new LinkedList<>(); }
@Override public void train(Dataset dataset) { params.svm_type = getLibsvmType(); svm_problem problem = toSvmProblem(dataset); ExecutorService service = newCachedThreadPool(threadsNamed("libsvm-trainer-" + System.identityHashCode(this) + "-%s")); try { TimeLimiter limiter = SimpleTimeLimiter.create(service); //TODO: this time limit should be configurable model = limiter.callWithTimeout(getTrainingFunction(problem, params), 1, TimeUnit.HOURS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException(e); } catch (ExecutionException e) { Throwable cause = e.getCause(); if (cause != null) { throwIfUnchecked(cause); throw new RuntimeException(cause); } } catch (Exception e) { throwIfUnchecked(e); throw new RuntimeException(e); } finally { service.shutdownNow(); } }
@Inject public DiscoveryNodeManager( @ServiceType("presto") ServiceSelector serviceSelector, NodeInfo nodeInfo, FailureDetector failureDetector, NodeVersion expectedNodeVersion, @ForNodeManager HttpClient httpClient, InternalCommunicationConfig internalCommunicationConfig) { this.serviceSelector = requireNonNull(serviceSelector, "serviceSelector is null"); this.failureDetector = requireNonNull(failureDetector, "failureDetector is null"); this.expectedNodeVersion = requireNonNull(expectedNodeVersion, "expectedNodeVersion is null"); this.httpClient = requireNonNull(httpClient, "httpClient is null"); this.nodeStateUpdateExecutor = newSingleThreadScheduledExecutor(threadsNamed("node-state-poller-%s")); this.nodeStateEventExecutor = newCachedThreadPool(threadsNamed("node-state-events-%s")); this.httpsRequired = internalCommunicationConfig.isHttpsRequired(); this.currentNode = findCurrentNode( serviceSelector.selectAllServices(), requireNonNull(nodeInfo, "nodeInfo is null").getNodeId(), expectedNodeVersion, httpsRequired); refreshNodesInternal(); }
this.executionFactories = requireNonNull(executionFactories, "executionFactories is null"); this.queryExecutor = newCachedThreadPool(threadsNamed("query-scheduler-%s")); this.queryExecutorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) queryExecutor); queryManagementExecutor = Executors.newScheduledThreadPool(queryManagerConfig.getQueryManagerExecutorPoolSize(), threadsNamed("query-management-%s")); queryManagementExecutorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) queryManagementExecutor);
taskNotificationExecutor = newFixedThreadPool(config.getTaskNotificationThreads(), threadsNamed("task-notification-%s")); taskNotificationExecutorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) taskNotificationExecutor); this.driverYieldExecutor = newScheduledThreadPool(config.getTaskYieldThreads(), threadsNamed("task-yield-%s"));
public TestSqlTask() { taskExecutor = new TaskExecutor(8, 16, 3, 4, Ticker.systemTicker()); taskExecutor.start(); taskNotificationExecutor = newScheduledThreadPool(10, threadsNamed("task-notification-%s")); driverYieldExecutor = newScheduledThreadPool(2, threadsNamed("driver-yield-%s")); LocalExecutionPlanner planner = createTestingPlanner(); sqlTaskExecutionFactory = new SqlTaskExecutionFactory( taskNotificationExecutor, taskExecutor, planner, createTestSplitMonitor(), new TaskManagerConfig()); }
.toInstance(newSingleThreadScheduledExecutor(threadsNamed("stage-scheduler"))); .toInstance(newCachedThreadPool(threadsNamed("query-execution-%s"))); binder.bind(QueryExecutionMBean.class).in(Scopes.SINGLETON); newExporter(binder).export(QueryExecutionMBean.class).as(generatedNameOf(QueryExecution.class));
@BeforeMethod public void setUp() { memoryPool = new MemoryPool(GENERAL_POOL, new DataSize(10, BYTE)); TaskExecutor taskExecutor = new TaskExecutor(8, 16, 3, 4, Ticker.systemTicker()); taskExecutor.start(); // Must be single threaded executor = newScheduledThreadPool(1, threadsNamed("task-notification-%s")); scheduledExecutor = newScheduledThreadPool(2, threadsNamed("task-notification-%s")); LocalExecutionPlanner planner = createTestingPlanner(); sqlTaskExecutionFactory = new SqlTaskExecutionFactory( executor, taskExecutor, planner, createTestSplitMonitor(), new TaskManagerConfig()); allOperatorContexts = null; }
throws Exception ScheduledExecutorService taskNotificationExecutor = newScheduledThreadPool(10, threadsNamed("task-notification-%s")); ScheduledExecutorService driverYieldExecutor = newScheduledThreadPool(2, threadsNamed("driver-yield-%s")); TaskExecutor taskExecutor = new TaskExecutor(5, 10, 3, 4, Ticker.systemTicker()); taskExecutor.start();
throws Exception ScheduledExecutorService taskNotificationExecutor = newScheduledThreadPool(10, threadsNamed("task-notification-%s")); ScheduledExecutorService driverYieldExecutor = newScheduledThreadPool(2, threadsNamed("driver-yield-%s")); TaskExecutor taskExecutor = new TaskExecutor(5, 10, 3, 4, Ticker.systemTicker()); taskExecutor.start();
public TaskManagementExecutor() { taskManagementExecutor = newScheduledThreadPool(5, threadsNamed("task-management-%s")); taskManagementExecutorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) taskManagementExecutor); }
public TaskManagementExecutor() { taskManagementExecutor = newScheduledThreadPool(5, threadsNamed("task-management-%s")); taskManagementExecutorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) taskManagementExecutor); }
@VisibleForTesting public TaskExecutor(int runnerThreads, int minDrivers, Ticker ticker) { checkArgument(runnerThreads > 0, "runnerThreads must be at least 1"); // we manages thread pool size directly, so create an unlimited pool this.executor = newCachedThreadPool(threadsNamed("task-processor-%s")); this.executorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) executor); this.runnerThreads = runnerThreads; this.ticker = requireNonNull(ticker, "ticker is null"); this.minimumNumberOfDrivers = minDrivers; this.pendingSplits = new PriorityBlockingQueue<>(Runtime.getRuntime().availableProcessors() * 10); this.tasks = new LinkedList<>(); }
@BeforeClass public final void setUp() throws Exception { executor = Executors.newCachedThreadPool(threadsNamed("test-%s")); }
@BeforeClass public final void setUp() throws Exception { executor = Executors.newCachedThreadPool(threadsNamed("test-%s")); }
@BeforeClass public final void setUp() throws Exception { executor = Executors.newCachedThreadPool(threadsNamed("test-%s")); }
public synchronized TestingSocksProxy start() throws IOException { checkState(serverSocket == null, "%s already started", getClass().getName()); try { serverSocket = new ServerSocket(bindPort, 50, InetAddress.getByName("127.0.0.1")); hostAndPort = HostAndPort.fromParts(serverSocket.getInetAddress().getHostAddress(), serverSocket.getLocalPort()); executorService = listeningDecorator(newCachedThreadPool(threadsNamed("socks-proxy-" + serverSocket.getLocalPort() + "-%s"))); executorService.execute(new SocksProxyAcceptor(serverSocket, executorService)); return this; } catch (Throwable e) { close(); throw e; } }
public TestSqlTask() { taskExecutor = new TaskExecutor(8, 16); taskExecutor.start(); taskNotificationExecutor = newScheduledThreadPool(10, threadsNamed("task-notification-%s")); LocalExecutionPlanner planner = createTestingPlanner(); sqlTaskExecutionFactory = new SqlTaskExecutionFactory( taskNotificationExecutor, taskExecutor, planner, new QueryMonitor(new ObjectMapperProvider().get(), new NullEventClient(), new NodeInfo("test"), new NodeVersion("testVersion")), new TaskManagerConfig()); }