/** * Get a new {@link java.util.concurrent.ThreadFactory} that uses a {@link LoggingUncaughtExceptionHandler} * to handle uncaught exceptions and the given thread name format. * * @param logger an {@link com.google.common.base.Optional} wrapping the {@link org.slf4j.Logger} that the * {@link LoggingUncaughtExceptionHandler} uses to log uncaught exceptions thrown in threads * @param nameFormat an {@link com.google.common.base.Optional} wrapping a thread naming format * @return a new {@link java.util.concurrent.ThreadFactory} */ public static ThreadFactory newThreadFactory(Optional<Logger> logger, Optional<String> nameFormat) { return newThreadFactory(new ThreadFactoryBuilder(), logger, nameFormat); }
/** * Shutdown an {@link ExecutorService} gradually, first disabling new task submissions and * later cancelling existing tasks. * * <p> * This method calls {@link #shutdownExecutorService(ExecutorService, Optional, long, TimeUnit)} * with default timeout time {@link #EXECUTOR_SERVICE_SHUTDOWN_TIMEOUT} and time unit * {@link #EXECUTOR_SERVICE_SHUTDOWN_TIMEOUT_TIMEUNIT}. * </p> * * @param executorService the {@link ExecutorService} to shutdown * @param logger an {@link Optional} wrapping a {@link Logger} to be used during shutdown */ public static void shutdownExecutorService(ExecutorService executorService, Optional<Logger> logger) { shutdownExecutorService(executorService, logger, EXECUTOR_SERVICE_SHUTDOWN_TIMEOUT, EXECUTOR_SERVICE_SHUTDOWN_TIMEOUT_TIMEUNIT); }
public ParallelRunner(int threads, FileSystem fs, FailPolicy failPolicy) { this.executor = ExecutorsUtils.loggingDecorator(Executors.newFixedThreadPool(threads, ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("ParallelRunner")))); this.fs = fs; this.failPolicy = failPolicy; }
/** * Start the {@link TaskScheduler}. * * @param name the name of the {@link TaskScheduler} */ @Override final void startImpl(Optional<String> name) { this.executorService = ExecutorsUtils.loggingDecorator(Executors.newScheduledThreadPool(0, ExecutorsUtils.newDaemonThreadFactory(Optional.of(LOGGER), name))); }
private void copyDataFiles(final Path outputDirectory, List<Path> inputFilePaths) throws IOException { ExecutorService executor = ScalingThreadPoolExecutor.newScalingThreadPool(0, this.copyLateDataThreadPoolSize, 100, ExecutorsUtils.newThreadFactory(Optional.of(LOG), Optional.of(this.dataset.getName() + "-copy-data"))); throw new IOException("Failed to copy file.", e); } finally { ExecutorsUtils.shutdownExecutorService(executor, Optional.of(LOG));
@Override public void startImpl() { this.executor = Executors.newSingleThreadScheduledExecutor( ExecutorsUtils.newDaemonThreadFactory(Optional.of(log), Optional.of("metrics-" + name + "-scheduler"))); this.reportingPeriodSeconds = parsePeriodToSeconds( config.hasPath(REPORTING_INTERVAL) ? config.getString(REPORTING_INTERVAL) : DEFAULT_REPORTING_INTERVAL_PERIOD); ensureMetricFilterIsInitialized(config); this.scheduledTask = Optional.<ScheduledFuture>of(this.executor.scheduleAtFixedRate(new Runnable() { @Override public void run() { report(); } }, 0, this.reportingPeriodSeconds, TimeUnit.SECONDS)); }
public IteratorExecutor(Iterator<Callable<T>> runnableIterator, int numThreads, ThreadFactory threadFactory) { this.numThreads = numThreads; this.iterator = runnableIterator; this.executor = ExecutorsUtils.loggingDecorator(Executors.newFixedThreadPool(numThreads, threadFactory)); this.completionService = new ExecutorCompletionService<>(this.executor); this.executed = false; }
(ThreadPoolExecutor) Executors.newFixedThreadPool(threadCount, ExecutorsUtils.newThreadFactory(logger)), 2, TimeUnit.MINUTES); ExecutorsUtils.shutdownExecutorService(executorService, logger, timeoutInSecs, TimeUnit.SECONDS);
protected void executeItertorExecutor(Iterator<Callable<Void>> callableIterator) throws IOException { try { IteratorExecutor<Void> executor = new IteratorExecutor<>(callableIterator, this.threadPoolSize, ExecutorsUtils.newDaemonThreadFactory(Optional.of(log), Optional.of(this.getClass().getSimpleName()))); List<Either<Void, ExecutionException>> results = executor.executeAndGetResults(); IteratorExecutor.logFailures(results, log, 10); } catch (InterruptedException ie) { throw new IOException("Dataset finder is interrupted.", ie); } }
private ListeningExecutorService getExecutorService() { return ExecutorsUtils.loggingDecorator( ScalingThreadPoolExecutor.newScalingThreadPool(0, this.threadPoolSize, TimeUnit.SECONDS.toMillis(10))); }
/** * Get a new {@link java.util.concurrent.ThreadFactory} that uses a {@link LoggingUncaughtExceptionHandler} * to handle uncaught exceptions. * * @param logger an {@link com.google.common.base.Optional} wrapping the {@link org.slf4j.Logger} that the * {@link LoggingUncaughtExceptionHandler} uses to log uncaught exceptions thrown in threads * @return a new {@link java.util.concurrent.ThreadFactory} */ public static ThreadFactory newThreadFactory(Optional<Logger> logger) { return newThreadFactory(logger, Optional.<String>absent()); }
public AbstractTaskStateTracker(int coreThreadPoolSize, Logger logger) { Preconditions.checkArgument(coreThreadPoolSize > 0, "Thread pool size should be positive"); this.taskMetricsUpdaterExecutor = ExecutorsUtils.loggingDecorator( new ScheduledThreadPoolExecutor(coreThreadPoolSize, ExecutorsUtils.newThreadFactory(Optional.of(logger), Optional.of("TaskStateTracker-%d")))); this.logger = logger; }
@Override public void close() throws IOException { ExecutorsUtils.shutdownExecutorService(_execService, Optional.of(_log), 5, TimeUnit.SECONDS); }
ExecutorsUtils.newThreadFactory(Optional.of(log), Optional.of("rename-thread-%d"))); Queue<Future<?>> futures = Queues.newConcurrentLinkedQueue(); ExecutorsUtils.shutdownExecutorService(executorService, Optional.of(log), 1, TimeUnit.SECONDS);
List<Either<Void, ExecutionException>> results = new IteratorExecutor<>(callableIterator, this.threadPoolOfGettingDatasetState, ExecutorsUtils.newDaemonThreadFactory(Optional.of(LOGGER), Optional.of("GetFsDatasetStateStore-"))) .executeAndGetResults(); int maxNumberOfErrorLogs = 10;
/** * Get a new {@link ThreadFactory} that uses a {@link LoggingUncaughtExceptionHandler} * to handle uncaught exceptions, uses the given thread name format, and produces daemon threads. * * @param logger an {@link Optional} wrapping the {@link Logger} that the * {@link LoggingUncaughtExceptionHandler} uses to log uncaught exceptions thrown in threads * @param nameFormat an {@link Optional} wrapping a thread naming format * @return a new {@link ThreadFactory} */ public static ThreadFactory newDaemonThreadFactory(Optional<Logger> logger, Optional<String> nameFormat) { return newThreadFactory(new ThreadFactoryBuilder().setDaemon(true), logger, nameFormat); }
public AsyncTrash(FileSystem fs, Properties properties, String user) throws IOException { int maxDeletingThreads = DEFAULT_MAX_DELETING_THREADS; if (properties.containsKey(MAX_DELETING_THREADS_KEY)) { maxDeletingThreads = Integer.parseInt(properties.getProperty(MAX_DELETING_THREADS_KEY)); } this.innerTrash = TrashFactory.createProxiedTrash(fs, properties, user); this.executor = ExecutorsUtils.loggingDecorator( MoreExecutors.getExitingExecutorService(ScalingThreadPoolExecutor.newScalingThreadPool(0, maxDeletingThreads, 100, ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("Async-trash-delete-pool-%d"))))); }
@Override protected void shutDown() throws Exception { ExecutorsUtils.shutdownExecutorService(this.fetchJobSpecExecutor, Optional.of(LOGGER)); } }
List<Either<Void, ExecutionException>> executionResults = new IteratorExecutor<>(callableIterator, this.getTaskCommitThreadPoolSize(), ExecutorsUtils.newDaemonThreadFactory(Optional.of(log), Optional.of("Task-committing-pool-%d"))) .executeAndGetResults(); IteratorExecutor.logFailures(executionResults, log, 10);
public TimeBasedLimiter(long timeLimit, TimeUnit timeUnit) { this.timeLimit = timeLimit; this.timeUnit = timeUnit; this.flagFlippingExecutor = new ScheduledThreadPoolExecutor(1, ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("TimeBasedThrottler"))); }