private void getServices() { final Properties properties = ConfigUtils.configToProperties(this.clusterConfig); this.taskExecutor = new TaskExecutor(properties); this.taskStateTracker = new GobblinHelixTaskStateTracker(properties); final List<Service> services = Lists.newArrayList(this.taskExecutor, this.taskStateTracker); this.serviceManager = new ServiceManager(services); }
@Override public void registerNewTask(Task task) { try { this.scheduledReporters.put(task.getTaskId(), scheduleTaskMetricsUpdater(new TaskMetricsUpdater(task), task)); } catch (RejectedExecutionException ree) { LOGGER.error(String.format("Scheduling of task state reporter for task %s was rejected", task.getTaskId())); } }
@AfterClass public void tearDown() throws IOException { try { if (this.localFs.exists(this.appWorkDir)) { this.localFs.delete(this.appWorkDir, true); } } finally { this.taskExecutor.stopAsync().awaitTerminated(); this.taskStateTracker.stopAsync().awaitTerminated(); } }
@BeforeClass public void setUp() throws IOException { Configuration configuration = new Configuration(); configuration.setInt(ConfigurationKeys.TASK_EXECUTOR_THREADPOOL_SIZE_KEY, 1); this.taskExecutor = new TaskExecutor(configuration); this.helixManager = Mockito.mock(HelixManager.class); Mockito.when(this.helixManager.getInstanceName()).thenReturn(GobblinHelixTaskTest.class.getSimpleName()); this.taskStateTracker = new GobblinHelixTaskStateTracker(new Properties()); this.localFs = FileSystem.getLocal(configuration); this.appWorkDir = new Path(GobblinHelixTaskTest.class.getSimpleName()); this.taskOutputDir = new Path(this.appWorkDir, "output"); }
@Override public void registerNewTask(Task task) { try { this.scheduledReporters.put(task.getTaskId(), scheduleTaskMetricsUpdater(new TaskMetricsUpdater(task), task)); } catch (RejectedExecutionException ree) { LOGGER.error(String.format("Scheduling of task state reporter for task %s was rejected", task.getTaskId())); } }
private GobblinHelixTaskFactory generateTaskFactory(TaskExecutor taskExecutor, Builder builder) { Properties properties = ConfigUtils.configToProperties(builder.getConfig()); URI rootPathUri = PathUtils.getRootPath(builder.getAppWorkPath()).toUri(); Config stateStoreJobConfig = ConfigUtils.propertiesToConfig(properties) .withValue(ConfigurationKeys.STATE_STORE_FS_URI_KEY, ConfigValueFactory.fromAnyRef(rootPathUri.toString())); TaskStateTracker taskStateTracker = new GobblinHelixTaskStateTracker(properties); services.add(taskExecutor); services.add(taskStateTracker); services.add(new JMXReportingService( ImmutableMap.of("task.executor", taskExecutor.getTaskExecutorQueueMetricSet()))); return new GobblinHelixTaskFactory(builder, taskExecutor, taskStateTracker, stateStoreJobConfig); } }
private void getServices() { final Properties properties = ConfigUtils.configToProperties(this.clusterConfig); this.taskExecutor = new TaskExecutor(properties); this.taskStateTracker = new GobblinHelixTaskStateTracker(properties); final List<Service> services = Lists.newArrayList(this.taskExecutor, this.taskStateTracker); this.serviceManager = new ServiceManager(services); }
private GobblinHelixTaskFactory generateTaskFactory(TaskExecutor taskExecutor, Builder builder) { Properties properties = ConfigUtils.configToProperties(builder.getConfig()); URI rootPathUri = PathUtils.getRootPath(builder.getAppWorkPath()).toUri(); Config stateStoreJobConfig = ConfigUtils.propertiesToConfig(properties) .withValue(ConfigurationKeys.STATE_STORE_FS_URI_KEY, ConfigValueFactory.fromAnyRef(rootPathUri.toString())); TaskStateTracker taskStateTracker = new GobblinHelixTaskStateTracker(properties); services.add(taskExecutor); services.add(taskStateTracker); services.add(new JMXReportingService( ImmutableMap.of("task.executor", taskExecutor.getTaskExecutorQueueMetricSet()))); return new GobblinHelixTaskFactory(builder.getContainerMetrics(), taskExecutor, taskStateTracker, builder.getFs(), builder.getAppWorkPath(), stateStoreJobConfig, builder.getHelixManager()); } }