/** * Get a {@link GobblinMetrics} instance with the given ID. * * @param id the given {@link GobblinMetrics} ID * @return a {@link GobblinMetrics} instance */ public static GobblinMetrics get(String id) { return get(id, null); }
@Test public void testInstrumented() { GobblinMetrics gobblinMetrics = GobblinMetrics.get("parent.context"); State state = new State(); state.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(true)); state.setProp(Instrumented.METRIC_CONTEXT_NAME_KEY, gobblinMetrics.getName()); Instrumented instrumented = new Instrumented(state, InstrumentedExtractor.class); Assert.assertNotNull(instrumented.getMetricContext()); Assert.assertTrue(instrumented.getMetricContext().getParent().isPresent()); Assert.assertEquals(instrumented.getMetricContext().getParent().get(), gobblinMetrics.getMetricContext()); Map<String, ?> tags = instrumented.getMetricContext().getTagMap(); Map<String, String> expectedTags = new HashMap<>(); expectedTags.put("construct", Constructs.EXTRACTOR.toString()); expectedTags.put("class", InstrumentedExtractor.class.getCanonicalName()); expectedTags.put(MetricContext.METRIC_CONTEXT_ID_TAG_NAME, tags.get(MetricContext.METRIC_CONTEXT_ID_TAG_NAME).toString()); expectedTags.put(MetricContext.METRIC_CONTEXT_NAME_TAG_NAME, tags.get(MetricContext.METRIC_CONTEXT_NAME_TAG_NAME).toString()); Assert.assertEquals(tags.size(), expectedTags.size()); for (Map.Entry<String, ?> tag : tags.entrySet()) { Assert.assertTrue(expectedTags.containsKey(tag.getKey())); Assert.assertEquals(expectedTags.get(tag.getKey()), tag.getValue().toString()); } }
.get(getForkMetricsName(taskContext.getTaskMetrics(), this.taskState, index), taskContext.getTaskMetrics().getMetricContext(), getForkMetricsTags(this.taskState, index)); this.closer.register(forkMetrics.getMetricContext());
public MRCompactor(Properties props, List<? extends Tag<?>> tags, Optional<CompactorListener> compactorListener) throws IOException { this.state = new State(); this.state.addAll(props); this.initilizeTime = getCurrentTime(); this.tags = tags; this.conf = HadoopUtils.getConfFromState(this.state); this.tmpOutputDir = getTmpOutputDir(); this.fs = getFileSystem(); this.datasets = getDatasetsFinder().findDistinctDatasets(); this.jobExecutor = createJobExecutor(); this.jobRunnables = Maps.newConcurrentMap(); this.closer = Closer.create(); this.stopwatch = Stopwatch.createStarted(); this.gobblinMetrics = initializeMetrics(); this.eventSubmitter = new EventSubmitter.Builder( GobblinMetrics.get(this.state.getProp(ConfigurationKeys.JOB_NAME_KEY)).getMetricContext(), MRCompactor.COMPACTION_TRACKING_EVENTS_NAMESPACE).build(); this.compactorListener = compactorListener; this.dataVerifTimeoutMinutes = getDataVerifTimeoutMinutes(); this.compactionTimeoutMinutes = getCompactionTimeoutMinutes(); this.shouldVerifDataCompl = shouldVerifyDataCompleteness(); this.compactionCompleteListener = getCompactionCompleteListener(); this.verifier = this.shouldVerifDataCompl ? Optional.of(this.closer.register(new DataCompletenessVerifier(this.state))) : Optional.<DataCompletenessVerifier> absent(); this.shouldPublishDataIfCannotVerifyCompl = shouldPublishDataIfCannotVerifyCompl(); }
/** * Get a {@link GobblinMetrics} instance with the given ID and parent {@link MetricContext}. * * @param id the given {@link GobblinMetrics} ID * @param parentContext the given parent {@link MetricContext} * @return a {@link GobblinMetrics} instance */ public static GobblinMetrics get(String id, MetricContext parentContext) { return get(id, parentContext, Lists.<Tag<?>>newArrayList()); }
GobblinMetrics.get(this.dataset.jobProps().getProp(ConfigurationKeys.JOB_NAME_KEY)).getMetricContext(), MRCompactor.COMPACTION_TRACKING_EVENTS_NAMESPACE).build();
@Override protected void shutDown() throws Exception { GobblinMetrics.get(this.appId).stopMetricsReporting(); } }
public MRCompactor(Properties props, List<? extends Tag<?>> tags, Optional<CompactorListener> compactorListener) throws IOException { this.state = new State(); this.state.addAll(props); this.initilizeTime = getCurrentTime(); this.tags = tags; this.conf = HadoopUtils.getConfFromState(this.state); this.tmpOutputDir = getTmpOutputDir(); this.fs = getFileSystem(); this.datasets = getDatasetsFinder().findDistinctDatasets(); this.jobExecutor = createJobExecutor(); this.jobRunnables = Maps.newConcurrentMap(); this.closer = Closer.create(); this.stopwatch = Stopwatch.createStarted(); this.gobblinMetrics = initializeMetrics(); this.eventSubmitter = new EventSubmitter.Builder( GobblinMetrics.get(this.state.getProp(ConfigurationKeys.JOB_NAME_KEY)).getMetricContext(), MRCompactor.COMPACTION_TRACKING_EVENTS_NAMESPACE).build(); this.compactorListener = compactorListener; this.dataVerifTimeoutMinutes = getDataVerifTimeoutMinutes(); this.compactionTimeoutMinutes = getCompactionTimeoutMinutes(); this.shouldVerifDataCompl = shouldVerifyDataCompleteness(); this.compactionCompleteListener = getCompactionCompleteListener(); this.verifier = this.shouldVerifDataCompl ? Optional.of(this.closer.register(new DataCompletenessVerifier(this.state))) : Optional.<DataCompletenessVerifier> absent(); this.shouldPublishDataIfCannotVerifyCompl = shouldPublishDataIfCannotVerifyCompl(); }
@Override protected void startUp() throws Exception { GobblinMetrics.get(this.appId).startMetricReporting(this.properties); }
private GobblinMetrics buildGobblinMetrics() { // Create tags list ImmutableList.Builder<Tag<?>> tags = new ImmutableList.Builder<>(); tags.add(new Tag<>(GobblinClusterMetricTagNames.APPLICATION_ID, this.applicationId)); tags.add(new Tag<>(GobblinClusterMetricTagNames.APPLICATION_NAME, this.applicationName)); // Intialize Gobblin metrics and start reporters GobblinMetrics gobblinMetrics = GobblinMetrics.get(this.applicationId, null, tags.build()); gobblinMetrics.startMetricReporting(ConfigUtils.configToProperties(config)); return gobblinMetrics; }
public GobblinMetricsPlugin(GobblinInstanceDriver instance) { super(instance); this.metrics = GobblinMetrics.get(getInstance().getInstanceName()); }
private GobblinMetrics initializeMetrics() { ImmutableList.Builder<Tag<?>> tags = ImmutableList.builder(); tags.addAll(this.tags); tags.addAll(Tag.fromMap(ClusterNameTags.getClusterNameTags())); GobblinMetrics gobblinMetrics = GobblinMetrics.get(this.state.getProp(ConfigurationKeys.JOB_NAME_KEY), null, tags.build()); gobblinMetrics.startMetricReporting(this.state.getProperties()); return gobblinMetrics; }
/** * Test the {@link GobblinMetrics} instance is removed from {@link GobblinMetricsRegistry} when * it stops metrics reporting */ public void testStopReportingMetrics() { String id = getClass().getSimpleName() + "-" + System.currentTimeMillis(); GobblinMetrics gobblinMetrics = GobblinMetrics.get(id); gobblinMetrics.startMetricReporting(new Properties()); Assert.assertEquals(GobblinMetricsRegistry.getInstance().get(id).get(), gobblinMetrics); gobblinMetrics.stopMetricsReporting(); Assert.assertFalse(GobblinMetricsRegistry.getInstance().get(id).isPresent()); } }
/** * Get a {@link GobblinMetrics} instance with the given ID. * * @param id the given {@link GobblinMetrics} ID * @return a {@link GobblinMetrics} instance */ public static GobblinMetrics get(String id) { return get(id, null); }
/** * Get a {@link GobblinMetrics} instance with the given ID and parent {@link MetricContext}. * * @param id the given {@link GobblinMetrics} ID * @param parentContext the given parent {@link MetricContext} * @return a {@link GobblinMetrics} instance */ public static GobblinMetrics get(String id, MetricContext parentContext) { return get(id, parentContext, Lists.<Tag<?>>newArrayList()); }
@Override protected void shutDown() throws Exception { GobblinMetrics.get(this.appId).stopMetricsReporting(); } }
@Override protected void startUp() throws Exception { GobblinMetrics.get(this.appId).startMetricReporting(this.properties); }
private GobblinMetrics buildGobblinMetrics() { // Create tags list ImmutableList.Builder<Tag<?>> tags = new ImmutableList.Builder<>(); tags.add(new Tag<>(GobblinClusterMetricTagNames.APPLICATION_ID, this.applicationId)); tags.add(new Tag<>(GobblinClusterMetricTagNames.APPLICATION_NAME, this.applicationName)); // Intialize Gobblin metrics and start reporters GobblinMetrics gobblinMetrics = GobblinMetrics.get(this.applicationId, null, tags.build()); gobblinMetrics.startMetricReporting(ConfigUtils.configToProperties(config)); return gobblinMetrics; }
public GobblinMetricsPlugin(GobblinInstanceDriver instance) { super(instance); this.metrics = GobblinMetrics.get(getInstance().getInstanceName()); }
private GobblinMetrics initializeMetrics() { ImmutableList.Builder<Tag<?>> tags = ImmutableList.builder(); tags.addAll(this.tags); tags.addAll(Tag.fromMap(ClusterNameTags.getClusterNameTags())); GobblinMetrics gobblinMetrics = GobblinMetrics.get(this.state.getProp(ConfigurationKeys.JOB_NAME_KEY), null, tags.build()); gobblinMetrics.startMetricReporting(this.state.getProperties()); return gobblinMetrics; }