/** * Get additional {@link Tag}s required for any type of reporting. */ private List<? extends Tag<?>> getMetadataTags(String applicationName, String applicationId) { return Tag.fromMap( new ImmutableMap.Builder<String, Object>().put(GobblinClusterMetricTagNames.APPLICATION_NAME, applicationName) .put(GobblinClusterMetricTagNames.APPLICATION_ID, applicationId).build()); }
/** * Takes a {@link List} of {@link Tag}s and returns a new {@link List} with the original {@link Tag}s as well as any * additional {@link Tag}s returned by {@link ClusterNameTags#getClusterNameTags()}. * * @see ClusterNameTags */ private static List<Tag<?>> addClusterNameTags(List<? extends Tag<?>> tags) { return ImmutableList.<Tag<?>>builder().addAll(tags).addAll(Tag.fromMap(ClusterNameTags.getClusterNameTags())) .build(); }
private Compactor getCompactor(CompactorFactory compactorFactory, Optional<CompactorListener> compactorListener) { try { return compactorFactory .createCompactor(this.properties, Tag.fromMap(AzkabanTags.getAzkabanTags()), compactorListener); } catch (CompactorCreationException e) { throw new RuntimeException("Unable to create compactor", e); } }
public AzkabanGobblinDaemon(String jobId, Properties props) throws Exception { super(jobId, LOG); List<Tag<?>> tags = Lists.newArrayList(); tags.addAll(Tag.fromMap(AzkabanTags.getAzkabanTags())); RootMetricContext.get(tags); this.daemon = new SchedulerDaemon(props); }
private GobblinMetrics initializeMetrics() { ImmutableList.Builder<Tag<?>> tags = ImmutableList.builder(); tags.addAll(this.tags); tags.addAll(Tag.fromMap(ClusterNameTags.getClusterNameTags())); GobblinMetrics gobblinMetrics = GobblinMetrics.get(this.state.getProp(ConfigurationKeys.JOB_NAME_KEY), null, tags.build()); gobblinMetrics.startMetricReporting(this.state.getProperties()); return gobblinMetrics; }
public ComplianceJob(Properties properties) { this.properties = properties; ExecutorService executor = ScalingThreadPoolExecutor.newScalingThreadPool(0, Integer.parseInt(properties.getProperty(ComplianceConfigurationKeys.MAX_CONCURRENT_DATASETS, ComplianceConfigurationKeys.DEFAULT_MAX_CONCURRENT_DATASETS)), 100, ExecutorsUtils.newThreadFactory(Optional.<Logger>absent(), Optional.of("complaince-job-pool-%d"))); this.service = MoreExecutors.listeningDecorator(executor); this.closer = Closer.create(); List<Tag<?>> tags = Lists.newArrayList(); tags.addAll(Tag.fromMap(AzkabanTags.getAzkabanTags())); this.metricContext = this.closer.register(Instrumented.getMetricContext(new State(properties), ComplianceJob.class, tags)); this.isMetricEnabled = GobblinMetrics.isEnabled(properties); this.eventSubmitter = new EventSubmitter.Builder(this.metricContext, ComplianceEvents.NAMESPACE).build(); this.throwables = Lists.newArrayList(); }
this.jobsMapping = jobsMapping; this.appWorkDir = builder.getAppWorkPath(); this.metadataTags = Tag.fromMap(new ImmutableMap.Builder<String, Object>() .put(GobblinClusterMetricTagNames.APPLICATION_NAME, builder.getApplicationName()) .put(GobblinClusterMetricTagNames.APPLICATION_ID, builder.getApplicationId())
tags.addAll(Tag.fromMap(AzkabanTags.getAzkabanTags())); RootMetricContext.get(tags);
tags.addAll(Tag.fromMap(AzkabanTags.getAzkabanTags()));
clusterNameTags.addAll(Tag.fromMap(ClusterNameTags.getClusterNameTags())); GobblinMetrics.addCustomTagsToProperties(jobProps, clusterNameTags);
/** * Get additional {@link Tag}s required for any type of reporting. */ private List<? extends Tag<?>> getMetadataTags(String applicationName, String applicationId) { return Tag.fromMap( new ImmutableMap.Builder<String, Object>().put(GobblinClusterMetricTagNames.APPLICATION_NAME, applicationName) .put(GobblinClusterMetricTagNames.APPLICATION_ID, applicationId).build()); }
/** * Takes a {@link List} of {@link Tag}s and returns a new {@link List} with the original {@link Tag}s as well as any * additional {@link Tag}s returned by {@link ClusterNameTags#getClusterNameTags()}. * * @see ClusterNameTags */ private static List<Tag<?>> addClusterNameTags(List<? extends Tag<?>> tags) { return ImmutableList.<Tag<?>>builder().addAll(tags).addAll(Tag.fromMap(ClusterNameTags.getClusterNameTags())) .build(); }
private Compactor getCompactor(CompactorFactory compactorFactory, Optional<CompactorListener> compactorListener) { try { return compactorFactory .createCompactor(this.properties, Tag.fromMap(AzkabanTags.getAzkabanTags()), compactorListener); } catch (CompactorCreationException e) { throw new RuntimeException("Unable to create compactor", e); } }
public AzkabanGobblinDaemon(String jobId, Properties props) throws Exception { super(jobId, LOG); List<Tag<?>> tags = Lists.newArrayList(); tags.addAll(Tag.fromMap(AzkabanTags.getAzkabanTags())); RootMetricContext.get(tags); this.daemon = new SchedulerDaemon(props); }
private GobblinMetrics initializeMetrics() { ImmutableList.Builder<Tag<?>> tags = ImmutableList.builder(); tags.addAll(this.tags); tags.addAll(Tag.fromMap(ClusterNameTags.getClusterNameTags())); GobblinMetrics gobblinMetrics = GobblinMetrics.get(this.state.getProp(ConfigurationKeys.JOB_NAME_KEY), null, tags.build()); gobblinMetrics.startMetricReporting(this.state.getProperties()); return gobblinMetrics; }
public GobblinHelixJobTask (TaskCallbackContext context, StateStores stateStores, TaskRunnerSuiteBase.Builder builder, GobblinHelixJobLauncherMetrics launcherMetrics, GobblinHelixJobTaskMetrics jobTaskMetrics) { this.jobTaskMetrics = jobTaskMetrics; this.taskConfig = context.getTaskConfig(); this.sysConfig = builder.getConfig(); this.helixManager = builder.getHelixManager(); this.jobPlusSysConfig = ConfigUtils.configToProperties(sysConfig); this.jobLauncherListener = new GobblinHelixJobLauncherListener(launcherMetrics); Map<String, String> configMap = this.taskConfig.getConfigMap(); for (Map.Entry<String, String> entry: configMap.entrySet()) { if (entry.getKey().startsWith(GobblinHelixDistributeJobExecutionLauncher.JOB_PROPS_PREFIX)) { String key = entry.getKey().replaceFirst(GobblinHelixDistributeJobExecutionLauncher.JOB_PROPS_PREFIX, ""); jobPlusSysConfig.put(key, entry.getValue()); } } if (!jobPlusSysConfig.containsKey(GobblinClusterConfigurationKeys.PLANNING_ID_KEY)) { throw new RuntimeException("Job doesn't have planning ID"); } this.planningJobId = jobPlusSysConfig.getProperty(GobblinClusterConfigurationKeys.PLANNING_ID_KEY); this.stateStores = stateStores; this.appWorkDir = builder.getAppWorkPath(); this.metadataTags = Tag.fromMap(new ImmutableMap.Builder<String, Object>() .put(GobblinClusterMetricTagNames.APPLICATION_NAME, builder.getApplicationName()) .put(GobblinClusterMetricTagNames.APPLICATION_ID, builder.getApplicationId()) .build()); }
tags.addAll(Tag.fromMap(AzkabanTags.getAzkabanTags()));
tags.addAll(Tag.fromMap(AzkabanTags.getAzkabanTags())); RootMetricContext.get(tags);
clusterNameTags.addAll(Tag.fromMap(ClusterNameTags.getClusterNameTags())); GobblinMetrics.addCustomTagsToProperties(jobProps, clusterNameTags);