@Inject public IndexLookup(Connector connector, ColumnCardinalityCache cardinalityCache) { this.connector = requireNonNull(connector, "connector is null"); this.cardinalityCache = requireNonNull(cardinalityCache, "cardinalityCache is null"); // Create a bounded executor with a pool size at 4x number of processors this.coreExecutor = newCachedThreadPool(daemonThreadsNamed("cardinality-lookup-%s")); this.executorService = new BoundedExecutor(coreExecutor, 4 * Runtime.getRuntime().availableProcessors()); }
@Test public void testExecutorCorruptionDetection() throws Exception { AtomicBoolean reject = new AtomicBoolean(); Executor executor = command -> { if (reject.get()) { throw new RejectedExecutionException(); } executorService.execute(command); }; BoundedExecutor boundedExecutor = new BoundedExecutor(executor, 1); // Enforce single thread // Force the underlying executor to fail reject.set(true); try { boundedExecutor.execute(() -> fail("Should not be run")); fail("Execute should fail"); } catch (Exception e) { } // Recover the underlying executor, but all new tasks should fail reject.set(false); try { boundedExecutor.execute(() -> fail("Should not be run")); fail("Execute should still fail"); } catch (Exception e) { } }
@Test public void testExecutorCorruptionDetection() throws Exception { AtomicBoolean reject = new AtomicBoolean(); Executor executor = command -> { if (reject.get()) { throw new RejectedExecutionException(); } executorService.execute(command); }; BoundedExecutor boundedExecutor = new BoundedExecutor(executor, 1); // Enforce single thread // Force the underlying executor to fail reject.set(true); try { boundedExecutor.execute(() -> fail("Should not be run")); fail("Execute should fail"); } catch (Exception e) { } // Recover the underlying executor, but all new tasks should fail reject.set(false); try { boundedExecutor.execute(() -> fail("Should not be run")); fail("Execute should still fail"); } catch (Exception e) { } }
@Override public TransactionId beginTransaction(IsolationLevel isolationLevel, boolean readOnly, boolean autoCommitContext) { TransactionId transactionId = TransactionId.create(); BoundedExecutor executor = new BoundedExecutor(finishingExecutor, maxFinishingConcurrency); TransactionMetadata transactionMetadata = new TransactionMetadata(transactionId, isolationLevel, readOnly, autoCommitContext, catalogManager, executor); checkState(transactions.put(transactionId, transactionMetadata) == null, "Duplicate transaction ID: %s", transactionId); return transactionId; }
@Test public void testExecutorCorruptionDetection() throws Exception { AtomicBoolean reject = new AtomicBoolean(); Executor executor = command -> { if (reject.get()) { throw new RejectedExecutionException(); } executorService.execute(command); }; BoundedExecutor boundedExecutor = new BoundedExecutor(executor, 1); // Enforce single thread // Force the underlying executor to fail reject.set(true); try { boundedExecutor.execute(() -> fail("Should not be run")); fail("Execute should fail"); } catch (Exception e) { } // Recover the underlying executor, but all new tasks should fail reject.set(false); try { boundedExecutor.execute(() -> fail("Should not be run")); fail("Execute should still fail"); } catch (Exception e) { } }
@Provides @Singleton @ForAsyncHttp public static BoundedExecutor createAsyncHttpResponseExecutor(@ForAsyncHttp ExecutorService coreExecutor, TaskManagerConfig config) { return new BoundedExecutor(coreExecutor, config.getHttpResponseThreads()); }
private void testBound(final int maxThreads, int stageTasks) BoundedExecutor boundedExecutor = new BoundedExecutor(executorService, maxThreads); boundedExecutor.execute(() -> { try { initializeLatch.countDown(); boundedExecutor.execute(() -> { try { int count = activeThreadCount.incrementAndGet();
@Provides @Singleton @ForStatementResource public static BoundedExecutor createStatementResponseExecutor(@ForStatementResource ExecutorService coreExecutor, TaskManagerConfig config) { return new BoundedExecutor(coreExecutor, config.getHttpResponseThreads()); }
private void testBound(final int maxThreads, int stageTasks) BoundedExecutor boundedExecutor = new BoundedExecutor(executorService, maxThreads); boundedExecutor.execute(() -> { try { initializeLatch.countDown(); boundedExecutor.execute(() -> { try { int count = activeThreadCount.incrementAndGet();
@Inject public ColumnCardinalityCache(Connector connector, AccumuloConfig config) { this.connector = requireNonNull(connector, "connector is null"); int size = requireNonNull(config, "config is null").getCardinalityCacheSize(); Duration expireDuration = config.getCardinalityCacheExpiration(); // Create a bounded executor with a pool size at 4x number of processors this.coreExecutor = newCachedThreadPool(daemonThreadsNamed("cardinality-lookup-%s")); this.executorService = new BoundedExecutor(coreExecutor, 4 * Runtime.getRuntime().availableProcessors()); LOG.debug("Created new cache size %d expiry %s", size, expireDuration); cache = CacheBuilder.newBuilder() .maximumSize(size) .expireAfterWrite(expireDuration.toMillis(), MILLISECONDS) .build(new CardinalityCacheLoader()); }
private void testBound(final int maxThreads, int stageTasks) BoundedExecutor boundedExecutor = new BoundedExecutor(executorService, maxThreads); boundedExecutor.execute(() -> { try { initializeLatch.countDown(); boundedExecutor.execute(() -> { try { int count = activeThreadCount.incrementAndGet();
@Inject public HttpRemoteTaskFactory(QueryManagerConfig config, TaskManagerConfig taskConfig, @ForScheduler HttpClient httpClient, LocationFactory locationFactory, JsonCodec<TaskStatus> taskStatusCodec, JsonCodec<TaskInfo> taskInfoCodec, JsonCodec<TaskUpdateRequest> taskUpdateRequestCodec, RemoteTaskStats stats) { this.httpClient = httpClient; this.locationFactory = locationFactory; this.taskStatusCodec = taskStatusCodec; this.taskInfoCodec = taskInfoCodec; this.taskUpdateRequestCodec = taskUpdateRequestCodec; this.maxErrorDuration = config.getRemoteTaskMaxErrorDuration(); this.taskStatusRefreshMaxWait = taskConfig.getStatusRefreshMaxWait(); this.taskInfoUpdateInterval = taskConfig.getInfoUpdateInterval(); this.coreExecutor = newCachedThreadPool(daemonThreadsNamed("remote-task-callback-%s")); this.executor = new BoundedExecutor(coreExecutor, config.getRemoteTaskMaxCallbackThreads()); this.executorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) coreExecutor); this.stats = requireNonNull(stats, "stats is null"); this.updateScheduledExecutor = newSingleThreadScheduledExecutor(daemonThreadsNamed("task-info-update-scheduler-%s")); this.errorScheduledExecutor = newSingleThreadScheduledExecutor(daemonThreadsNamed("remote-task-error-delay-%s")); }
renameExecution = new BoundedExecutor(executorService, maxConcurrentFileRenames);
hdfsEnvironment, directoryLister, new BoundedExecutor(executorService, hiveClientConfig.getMaxSplitIteratorThreads()), coercionPolicy, new CounterStat(),
hdfsEnvironment, new HadoopDirectoryLister(), new BoundedExecutor(executor, config.getMaxSplitIteratorThreads()), new HiveCoercionPolicy(TYPE_MANAGER), new CounterStat(),
public TransactionId beginTransaction(IsolationLevel isolationLevel, boolean readOnly, boolean autoCommitContext) { TransactionId transactionId = TransactionId.create(); BoundedExecutor executor = new BoundedExecutor(finishingExecutor, maxFinishingConcurrency); TransactionMetadata transactionMetadata = new TransactionMetadata(transactionId, isolationLevel, readOnly, autoCommitContext, executor); checkState(transactions.put(transactionId, transactionMetadata) == null, "Duplicate transaction ID: %s", transactionId); return transactionId; }
@Override public TransactionId beginTransaction(IsolationLevel isolationLevel, boolean readOnly, boolean autoCommitContext) { TransactionId transactionId = TransactionId.create(); BoundedExecutor executor = new BoundedExecutor(finishingExecutor, maxFinishingConcurrency); TransactionMetadata transactionMetadata = new TransactionMetadata(transactionId, isolationLevel, readOnly, autoCommitContext, catalogManager, executor); checkState(transactions.put(transactionId, transactionMetadata) == null, "Duplicate transaction ID: %s", transactionId); return transactionId; }
@Override public TransactionId beginTransaction(IsolationLevel isolationLevel, boolean readOnly, boolean autoCommitContext) { TransactionId transactionId = TransactionId.create(); BoundedExecutor executor = new BoundedExecutor(finishingExecutor, maxFinishingConcurrency); TransactionMetadata transactionMetadata = new TransactionMetadata(transactionId, isolationLevel, readOnly, autoCommitContext, catalogManager, executor); checkState(transactions.put(transactionId, transactionMetadata) == null, "Duplicate transaction ID: %s", transactionId); return transactionId; }
@Provides @Singleton @ForStatementResource public static BoundedExecutor createStatementResponseExecutor(@ForStatementResource ExecutorService coreExecutor, TaskManagerConfig config) { return new BoundedExecutor(coreExecutor, config.getHttpResponseThreads()); }
@Provides @Singleton @ForAsyncHttp public static BoundedExecutor createAsyncHttpResponseExecutor(@ForAsyncHttp ExecutorService coreExecutor, TaskManagerConfig config) { return new BoundedExecutor(coreExecutor, config.getHttpResponseThreads()); }