@Inject public IndexLookup(Connector connector, ColumnCardinalityCache cardinalityCache) { this.connector = requireNonNull(connector, "connector is null"); this.cardinalityCache = requireNonNull(cardinalityCache, "cardinalityCache is null"); // Create a bounded executor with a pool size at 4x number of processors this.coreExecutor = newCachedThreadPool(daemonThreadsNamed("cardinality-lookup-%s")); this.executorService = new BoundedExecutor(coreExecutor, 4 * Runtime.getRuntime().availableProcessors()); }
@Override public TransactionId beginTransaction(IsolationLevel isolationLevel, boolean readOnly, boolean autoCommitContext) { TransactionId transactionId = TransactionId.create(); BoundedExecutor executor = new BoundedExecutor(finishingExecutor, maxFinishingConcurrency); TransactionMetadata transactionMetadata = new TransactionMetadata(transactionId, isolationLevel, readOnly, autoCommitContext, catalogManager, executor); checkState(transactions.put(transactionId, transactionMetadata) == null, "Duplicate transaction ID: %s", transactionId); return transactionId; }
@Provides @Singleton @ForAsyncHttp public static BoundedExecutor createAsyncHttpResponseExecutor(@ForAsyncHttp ExecutorService coreExecutor, TaskManagerConfig config) { return new BoundedExecutor(coreExecutor, config.getHttpResponseThreads()); }
@Provides @Singleton @ForStatementResource public static BoundedExecutor createStatementResponseExecutor(@ForStatementResource ExecutorService coreExecutor, TaskManagerConfig config) { return new BoundedExecutor(coreExecutor, config.getHttpResponseThreads()); }
@Inject public ColumnCardinalityCache(Connector connector, AccumuloConfig config) { this.connector = requireNonNull(connector, "connector is null"); int size = requireNonNull(config, "config is null").getCardinalityCacheSize(); Duration expireDuration = config.getCardinalityCacheExpiration(); // Create a bounded executor with a pool size at 4x number of processors this.coreExecutor = newCachedThreadPool(daemonThreadsNamed("cardinality-lookup-%s")); this.executorService = new BoundedExecutor(coreExecutor, 4 * Runtime.getRuntime().availableProcessors()); LOG.debug("Created new cache size %d expiry %s", size, expireDuration); cache = CacheBuilder.newBuilder() .maximumSize(size) .expireAfterWrite(expireDuration.toMillis(), MILLISECONDS) .build(new CardinalityCacheLoader()); }
@Inject public HttpRemoteTaskFactory(QueryManagerConfig config, TaskManagerConfig taskConfig, @ForScheduler HttpClient httpClient, LocationFactory locationFactory, JsonCodec<TaskStatus> taskStatusCodec, JsonCodec<TaskInfo> taskInfoCodec, JsonCodec<TaskUpdateRequest> taskUpdateRequestCodec, RemoteTaskStats stats) { this.httpClient = httpClient; this.locationFactory = locationFactory; this.taskStatusCodec = taskStatusCodec; this.taskInfoCodec = taskInfoCodec; this.taskUpdateRequestCodec = taskUpdateRequestCodec; this.maxErrorDuration = config.getRemoteTaskMaxErrorDuration(); this.taskStatusRefreshMaxWait = taskConfig.getStatusRefreshMaxWait(); this.taskInfoUpdateInterval = taskConfig.getInfoUpdateInterval(); this.coreExecutor = newCachedThreadPool(daemonThreadsNamed("remote-task-callback-%s")); this.executor = new BoundedExecutor(coreExecutor, config.getRemoteTaskMaxCallbackThreads()); this.executorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) coreExecutor); this.stats = requireNonNull(stats, "stats is null"); this.updateScheduledExecutor = newSingleThreadScheduledExecutor(daemonThreadsNamed("task-info-update-scheduler-%s")); this.errorScheduledExecutor = newSingleThreadScheduledExecutor(daemonThreadsNamed("remote-task-error-delay-%s")); }
renameExecution = new BoundedExecutor(executorService, maxConcurrentFileRenames);
hdfsEnvironment, directoryLister, new BoundedExecutor(executorService, hiveClientConfig.getMaxSplitIteratorThreads()), coercionPolicy, new CounterStat(),
hdfsEnvironment, new HadoopDirectoryLister(), new BoundedExecutor(executor, config.getMaxSplitIteratorThreads()), new HiveCoercionPolicy(TYPE_MANAGER), new CounterStat(),
public TransactionId beginTransaction(IsolationLevel isolationLevel, boolean readOnly, boolean autoCommitContext) { TransactionId transactionId = TransactionId.create(); BoundedExecutor executor = new BoundedExecutor(finishingExecutor, maxFinishingConcurrency); TransactionMetadata transactionMetadata = new TransactionMetadata(transactionId, isolationLevel, readOnly, autoCommitContext, executor); checkState(transactions.put(transactionId, transactionMetadata) == null, "Duplicate transaction ID: %s", transactionId); return transactionId; }
@Override public TransactionId beginTransaction(IsolationLevel isolationLevel, boolean readOnly, boolean autoCommitContext) { TransactionId transactionId = TransactionId.create(); BoundedExecutor executor = new BoundedExecutor(finishingExecutor, maxFinishingConcurrency); TransactionMetadata transactionMetadata = new TransactionMetadata(transactionId, isolationLevel, readOnly, autoCommitContext, catalogManager, executor); checkState(transactions.put(transactionId, transactionMetadata) == null, "Duplicate transaction ID: %s", transactionId); return transactionId; }
@Override public TransactionId beginTransaction(IsolationLevel isolationLevel, boolean readOnly, boolean autoCommitContext) { TransactionId transactionId = TransactionId.create(); BoundedExecutor executor = new BoundedExecutor(finishingExecutor, maxFinishingConcurrency); TransactionMetadata transactionMetadata = new TransactionMetadata(transactionId, isolationLevel, readOnly, autoCommitContext, catalogManager, executor); checkState(transactions.put(transactionId, transactionMetadata) == null, "Duplicate transaction ID: %s", transactionId); return transactionId; }
@Provides @Singleton @ForStatementResource public static BoundedExecutor createStatementResponseExecutor(@ForStatementResource ExecutorService coreExecutor, TaskManagerConfig config) { return new BoundedExecutor(coreExecutor, config.getHttpResponseThreads()); }
@Provides @Singleton @ForAsyncHttp public static BoundedExecutor createAsyncHttpResponseExecutor(@ForAsyncHttp ExecutorService coreExecutor, TaskManagerConfig config) { return new BoundedExecutor(coreExecutor, config.getHttpResponseThreads()); }
@Provides @Singleton @ForAsyncHttp public static BoundedExecutor createAsyncHttpResponseExecutor(@ForAsyncHttp ExecutorService coreExecutor, TaskManagerConfig config) { return new BoundedExecutor(coreExecutor, config.getHttpResponseThreads()); }
private static <T> T timeLimited(T target, Class<T> clazz, Duration timeout, ExecutorService executor, int maxThreads) { executor = new ExecutorServiceAdapter(new BoundedExecutor(executor, maxThreads)); TimeLimiter limiter = SimpleTimeLimiter.create(executor); return limiter.newProxy(target, clazz, timeout.toMillis(), MILLISECONDS); }
@Provides @Singleton @ForAsyncHttp public static BoundedExecutor createAsyncHttpResponseExecutor(@ForAsyncHttp ExecutorService coreExecutor, TaskManagerConfig config) { return new BoundedExecutor(coreExecutor, config.getHttpResponseThreads()); }
@Provides @Singleton @ForStatementResource public static BoundedExecutor createStatementResponseExecutor(@ForStatementResource ExecutorService coreExecutor, TaskManagerConfig config) { return new BoundedExecutor(coreExecutor, config.getHttpResponseThreads()); }
private static <T> T timeLimited(T target, Class<T> clazz, Duration timeout, ExecutorService executor, int maxThreads) { executor = new ExecutorServiceAdapter(new BoundedExecutor(executor, maxThreads)); TimeLimiter limiter = SimpleTimeLimiter.create(executor); return limiter.newProxy(target, clazz, timeout.toMillis(), MILLISECONDS); }
@Inject public HttpRemoteTaskFactory(QueryManagerConfig config, TaskManagerConfig taskConfig, @ForScheduler HttpClient httpClient, LocationFactory locationFactory, JsonCodec<TaskInfo> taskInfoCodec, JsonCodec<TaskUpdateRequest> taskUpdateRequestCodec) { this.httpClient = httpClient; this.locationFactory = locationFactory; this.taskInfoCodec = taskInfoCodec; this.taskUpdateRequestCodec = taskUpdateRequestCodec; this.minErrorDuration = config.getRemoteTaskMinErrorDuration(); this.taskInfoRefreshMaxWait = taskConfig.getInfoRefreshMaxWait(); this.coreExecutor = newCachedThreadPool(daemonThreadsNamed("remote-task-callback-%s")); this.executor = new BoundedExecutor(coreExecutor, config.getRemoteTaskMaxCallbackThreads()); this.executorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) coreExecutor); this.errorScheduledExecutor = newSingleThreadScheduledExecutor(daemonThreadsNamed("remote-task-error-delay-%s")); }