/** * Get a new {@link java.util.concurrent.ThreadFactory} that uses a {@link LoggingUncaughtExceptionHandler} * to handle uncaught exceptions and the given thread name format. * * @param logger an {@link com.google.common.base.Optional} wrapping the {@link org.slf4j.Logger} that the * {@link LoggingUncaughtExceptionHandler} uses to log uncaught exceptions thrown in threads * @param nameFormat an {@link com.google.common.base.Optional} wrapping a thread naming format * @return a new {@link java.util.concurrent.ThreadFactory} */ public static ThreadFactory newThreadFactory(Optional<Logger> logger, Optional<String> nameFormat) { return newThreadFactory(new ThreadFactoryBuilder(), logger, nameFormat); }
/** * Get a new {@link java.util.concurrent.ThreadFactory} that uses a {@link LoggingUncaughtExceptionHandler} * to handle uncaught exceptions. * * @param logger an {@link com.google.common.base.Optional} wrapping the {@link org.slf4j.Logger} that the * {@link LoggingUncaughtExceptionHandler} uses to log uncaught exceptions thrown in threads * @return a new {@link java.util.concurrent.ThreadFactory} */ public static ThreadFactory newThreadFactory(Optional<Logger> logger) { return newThreadFactory(logger, Optional.<String>absent()); }
/** * Get a new {@link ThreadFactory} that uses a {@link LoggingUncaughtExceptionHandler} * to handle uncaught exceptions, uses the given thread name format, and produces daemon threads. * * @param logger an {@link Optional} wrapping the {@link Logger} that the * {@link LoggingUncaughtExceptionHandler} uses to log uncaught exceptions thrown in threads * @param nameFormat an {@link Optional} wrapping a thread naming format * @return a new {@link ThreadFactory} */ public static ThreadFactory newDaemonThreadFactory(Optional<Logger> logger, Optional<String> nameFormat) { return newThreadFactory(new ThreadFactoryBuilder().setDaemon(true), logger, nameFormat); }
public TimeBasedLimiter(long timeLimit, TimeUnit timeUnit) { this.timeLimit = timeLimit; this.timeUnit = timeUnit; this.flagFlippingExecutor = new ScheduledThreadPoolExecutor(1, ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("TimeBasedThrottler"))); }
public ParallelJobListener(List<JobListener> jobListeners) { this.jobListeners = jobListeners; this.executor = Executors.newCachedThreadPool( ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("ParallelJobListener"))); this.completionService = new ExecutorCompletionService<>(this.executor); }
public static ExecutorService getDefaultExecutor(Logger log) { return Executors.newSingleThreadExecutor( ExecutorsUtils.newThreadFactory(Optional.of(log), Optional.of(log.getName() + "-%d"))); }
protected void startAsync() throws JobException { _log.info("Starting " + getClass().getSimpleName()); ExecutorsUtils.newThreadFactory(Optional.of(_log), Optional.of("job-launcher-execution-driver")).newThread(this).start(); }
public ParallelRunner(int threads, FileSystem fs, FailPolicy failPolicy) { this.executor = ExecutorsUtils.loggingDecorator(Executors.newFixedThreadPool(threads, ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("ParallelRunner")))); this.fs = fs; this.failPolicy = failPolicy; }
public AbstractTaskStateTracker(int coreThreadPoolSize, Logger logger) { Preconditions.checkArgument(coreThreadPoolSize > 0, "Thread pool size should be positive"); this.taskMetricsUpdaterExecutor = ExecutorsUtils.loggingDecorator( new ScheduledThreadPoolExecutor(coreThreadPoolSize, ExecutorsUtils.newThreadFactory(Optional.of(logger), Optional.of("TaskStateTracker-%d")))); this.logger = logger; }
public AWSJobConfigurationManager(EventBus eventBus, Config config) { super(eventBus, config); this.jobConfFiles = Maps.newHashMap(); if (config.hasPath(GobblinAWSConfigurationKeys.JOB_CONF_REFRESH_INTERVAL)) { this.refreshIntervalInSeconds = config.getDuration(GobblinAWSConfigurationKeys.JOB_CONF_REFRESH_INTERVAL, TimeUnit.SECONDS); } else { this.refreshIntervalInSeconds = DEFAULT_JOB_CONF_REFRESH_INTERVAL; } this.fetchJobConfExecutor = Executors.newSingleThreadScheduledExecutor( ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("FetchJobConfExecutor"))); }
private synchronized ExecutorService getExecutorService() { if(!this.executorServiceOptional.isPresent()) { this.executorServiceOptional = Optional.of(MoreExecutors.getExitingExecutorService( (ThreadPoolExecutor) Executors.newCachedThreadPool(ExecutorsUtils.newThreadFactory(Optional.of(LOG), Optional.of("MetricContext-" + getName() + "-%d"))), 5, TimeUnit.MINUTES)); } return this.executorServiceOptional.get(); }
/** * Constructor used internally. */ private TaskExecutor(int taskExecutorThreadPoolSize, int coreRetryThreadPoolSize, long retryIntervalInSeconds) { Preconditions.checkArgument(taskExecutorThreadPoolSize > 0, "Task executor thread pool size should be positive"); Preconditions.checkArgument(retryIntervalInSeconds > 0, "Task retry interval should be positive"); // Currently a fixed-size thread pool is used to execute tasks. We probably need to revisit this later. this.taskExecutor = Executors.newScheduledThreadPool( taskExecutorThreadPoolSize, ExecutorsUtils.newThreadFactory(Optional.of(LOG), Optional.of("TaskExecutor-%d"))); this.retryIntervalInSeconds = retryIntervalInSeconds; this.forkExecutor = new ThreadPoolExecutor( // The core thread pool size is equal to that of the task executor as there's at least one fork per task taskExecutorThreadPoolSize, // The fork executor thread pool size is essentially unbounded. This is to make sure all forks of // a task get a thread to run so all forks of the task are making progress. This is necessary since // otherwise the parent task will be blocked if the record queue (bounded) of some fork is full and // that fork has not yet started to run because of no available thread. The task cannot proceed in // this case because it has to make sure every records go to every forks. Integer.MAX_VALUE, 0L, TimeUnit.MILLISECONDS, // The work queue is a SynchronousQueue. This essentially forces a new thread to be created for each fork. new SynchronousQueue<Runnable>(), ExecutorsUtils.newThreadFactory(Optional.of(LOG), Optional.of("ForkExecutor-%d"))); }
public StateStoreCleaner(Properties properties) throws IOException { Preconditions.checkArgument(properties.containsKey(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY), "Missing configuration property for the state store root directory: " + ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY); this.stateStoreRootDir = new Path(properties.getProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY)); this.retention = Long.parseLong( properties.getProperty(STATE_STORE_CLEANER_RETENTION_KEY, DEFAULT_STATE_STORE_CLEANER_RETENTION)); this.retentionTimeUnit = TimeUnit.valueOf(properties .getProperty(STATE_STORE_CLEANER_RETENTION_TIMEUNIT_KEY, DEFAULT_STATE_STORE_CLEANER_RETENTION_TIMEUNIT) .toUpperCase()); this.cleanerRunnerExecutor = Executors.newFixedThreadPool(Integer.parseInt(properties .getProperty(STATE_STORE_CLEANER_EXECUTOR_THREADS_KEY, DEFAULT_STATE_STORE_CLEANER_EXECUTOR_THREADS)), ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("StateStoreCleaner"))); URI fsUri = URI.create(properties.getProperty(ConfigurationKeys.STATE_STORE_FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI)); this.fs = FileSystem.get(fsUri, new Configuration()); }
public TimestampBasedCopyableDataset(FileSystem fs, Properties props, Path datasetRoot) { this.srcFs = fs; this.datasetRoot = datasetRoot; try { Class<?> copyPolicyClass = Class.forName(props.getProperty(COPY_POLICY, DEFAULT_COPY_POLICY)); this.versionSelectionPolicy = (VersionSelectionPolicy<TimestampedDatasetVersion>) copyPolicyClass.getConstructor(Properties.class) .newInstance(props); Class<?> timestampedDatasetVersionFinderClass = Class.forName(props.getProperty(DATASET_VERSION_FINDER, DEFAULT_DATASET_VERSION_FINDER)); this.datasetVersionFinder = (VersionFinder<TimestampedDatasetVersion>) timestampedDatasetVersionFinderClass.getConstructor( FileSystem.class, Properties.class).newInstance(this.srcFs, props); } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException exception) { throw new RuntimeException(exception); } this.executor = ScalingThreadPoolExecutor.newScalingThreadPool(0, Integer.parseInt(props.getProperty( THREADPOOL_SIZE_TO_GET_COPYABLE_FILES, DEFAULT_THREADPOOL_SIZE_TO_GET_COPYABLE_FILES)), 100, ExecutorsUtils .newThreadFactory(Optional.of(log), Optional.of(getClass().getSimpleName()))); }
public AsyncTrash(FileSystem fs, Properties properties, String user) throws IOException { int maxDeletingThreads = DEFAULT_MAX_DELETING_THREADS; if (properties.containsKey(MAX_DELETING_THREADS_KEY)) { maxDeletingThreads = Integer.parseInt(properties.getProperty(MAX_DELETING_THREADS_KEY)); } this.innerTrash = TrashFactory.createProxiedTrash(fs, properties, user); this.executor = ExecutorsUtils.loggingDecorator( MoreExecutors.getExitingExecutorService(ScalingThreadPoolExecutor.newScalingThreadPool(0, maxDeletingThreads, 100, ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("Async-trash-delete-pool-%d"))))); }
protected HiveRegister(State state) { this.props = new HiveRegProps(state); this.hiveDbRootDir = this.props.getDbRootDir(); this.executor = ExecutorsUtils.loggingDecorator( ScalingThreadPoolExecutor.newScalingThreadPool(0, this.props.getNumThreads(), TimeUnit.SECONDS.toMillis(10), ExecutorsUtils.newThreadFactory(Optional.of(log), Optional.of(getClass().getSimpleName())))); }
public EventReporter(Builder builder) { super(builder.context, builder.name, builder.filter, builder.rateUnit, builder.durationUnit); this.closer = Closer.create(); this.immediateReportExecutor = MoreExecutors.getExitingExecutorService( (ThreadPoolExecutor) Executors.newFixedThreadPool(1, ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("EventReporter-" + builder.name + "-%d"))), 5, TimeUnit.MINUTES); this.metricContext = builder.context; this.notificationTargetKey = builder.context.addNotificationTarget(new Function<Notification, Void>() { @Nullable @Override public Void apply(Notification notification) { notificationCallback(notification); return null; } }); this.reportingQueue = Queues.newLinkedBlockingQueue(QUEUE_CAPACITY); }
public ScheduledJobConfigurationManager(EventBus eventBus, Config config) { super(eventBus, config); this.jobSpecs = Maps.newHashMap(); this.refreshIntervalInSeconds = ConfigUtils.getLong(config, GobblinClusterConfigurationKeys.JOB_SPEC_REFRESH_INTERVAL, DEFAULT_JOB_SPEC_REFRESH_INTERVAL); this.fetchJobSpecExecutor = Executors.newSingleThreadScheduledExecutor( ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("FetchJobSpecExecutor"))); this.aliasResolver = new ClassAliasResolver<>(SpecExecutorInstanceConsumer.class); try { String specExecutorInstanceConsumerClassName = GobblinClusterConfigurationKeys.DEFAULT_SPEC_EXECUTOR_INSTANCE_CONSUMER_CLASS; if (config.hasPath(GobblinClusterConfigurationKeys.SPEC_EXECUTOR_INSTANCE_CONSUMER_CLASS_KEY)) { specExecutorInstanceConsumerClassName = config.getString(GobblinClusterConfigurationKeys.SPEC_EXECUTOR_INSTANCE_CONSUMER_CLASS_KEY); } LOGGER.info("Using SpecExecutorInstanceConsumer ClassNameclass name/alias " + specExecutorInstanceConsumerClassName); this.specExecutorInstanceConsumer = (SpecExecutorInstanceConsumer) ConstructorUtils .invokeConstructor(Class.forName(this.aliasResolver.resolve( specExecutorInstanceConsumerClassName)), config); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new RuntimeException(e); } }
@Override public void run() { try { JobExecutionDriver driver = _jobLauncher.launchJob(new ResolvedJobSpec(_jobSpec, _instanceDriver)); _callbacksDispatcher.onJobLaunch(driver); driver.registerStateListener(new JobStateTracker()); ExecutorsUtils.newThreadFactory(Optional.of(_log), Optional.of("gobblin-instance-driver")).newThread(driver).start(); } catch (Throwable t) { _log.error("Job launch failed: " + t, t); } } }
private RootMetricContext(List<Tag<?>> tags) throws NameConflictException { super(ROOT_METRIC_CONTEXT, null, tags, true); this.innerMetricContexts = Sets.newConcurrentHashSet(); this.referenceQueue = new ReferenceQueue<>(); this.referenceQueueExecutorService = ExecutorsUtils.loggingDecorator(MoreExecutors.getExitingScheduledExecutorService(new ScheduledThreadPoolExecutor(1, ExecutorsUtils.newThreadFactory(Optional.of(log), Optional.of("GobblinMetrics-ReferenceQueue"))))); this.referenceQueueExecutorService.scheduleWithFixedDelay(new CheckReferenceQueue(), 0, 2, TimeUnit.SECONDS); this.reporters = Sets.newConcurrentHashSet(); this.reportingStarted = false; addShutdownHook(); }