/** * Get a new {@link MetricContext.Builder} for building child {@link MetricContext}s. * * @param name name of the child {@link MetricContext} to be built * @return a new {@link MetricContext.Builder} for building child {@link MetricContext}s */ public Builder childBuilder(String name) { return builder(name).hasParent(this); }
: MetricContext.builder(newName); return builder.addTags(context.getTags()).addTags(newTags).build();
public ReporterExampleBase(ContextAwareScheduledReporter.Builder reporterBuilder, int tasks, long totalRecords) { this.executor = Executors.newFixedThreadPool(10); this.context = MetricContext.builder("Job") .addTag(new Tag<String>(JobEvent.METADATA_JOB_NAME, "ExampleJob")) .addTag(new Tag<String>(JobEvent.METADATA_JOB_ID, JOB_NAME + "_" + System.currentTimeMillis())) .build(); this.reporterBuilder = reporterBuilder; this.tasks = tasks; this.totalRecords = totalRecords; }
: MetricContext.builder(klazz.getCanonicalName() + "." + randomId); return builder.addTags(generatedTags).addTags(tags).build();
@Test public void testTags() throws IOException { MetricContext metricContext = MetricContext.builder(this.getClass().getCanonicalName()).build(); Counter counter = metricContext.counter("com.linkedin.example.counter"); Map<String, String> tags = new HashMap<>(); tags.put("testKey", "testValue"); tags.put("key2", "value2"); OutputStreamReporter reporter = OutputStreamReporter.Factory.newBuilder().withTags(tags).outputTo(this.stream).build(new Properties()); counter.inc(); reporter.report(); Assert.assertTrue(this.stream.toString().contains("key2=value2")); Assert.assertTrue(this.stream.toString().contains("testKey=testValue")); String[] lines = this.stream.toString().split("\n"); Map<String, Set<String>> expected = new HashMap<>(); expectMetrics(expected, lines); Set<String> counterSubMetrics = new HashSet<>(); counterSubMetrics.add("count"); expected.put("com.linkedin.example.counter", counterSubMetrics); reporter.close(); }
@BeforeClass public void setUp() { String contextName = CONTEXT_NAME + "_" + UUID.randomUUID().toString(); this.context = MetricContext.builder(contextName) .addTag(new Tag<String>(JOB_ID_KEY, JOB_ID_PREFIX + 0)) .build(); Assert.assertEquals(this.context.getName(), contextName); Assert.assertTrue(this.context.getParent().isPresent()); Assert.assertEquals(this.context.getParent().get(), RootMetricContext.get()); Assert.assertEquals(this.context.getTags().size(), 3); // uuid and name tag gets added automatically Assert.assertEquals(this.context.getTags().get(0).getKey(), JOB_ID_KEY); Assert.assertEquals(this.context.getTags().get(0).getValue(), JOB_ID_PREFIX + 0); // Second tag should be uuid Assert.assertTrue(this.context.getTags().get(1).getValue().toString() .matches("[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}")); }
@Test public void testReport() throws IOException { MetricContext testContext = MetricContext.builder(getClass().getCanonicalName()).build(); FileSystem fs = mock(FileSystem.class); Path failureLogPath = mock(Path.class);
@Test public void testTagsFromContext() throws IOException { Tag<?> tag1 = new Tag<>("tag1", "value1"); MetricContext context = MetricContext.builder("context").addTag(tag1).build(); Counter counter = context.counter("com.linkedin.example.counter"); OutputStreamReporter reporter = OutputStreamReporter.Factory.newBuilder().outputTo(this.stream).build(new Properties()); counter.inc(); reporter.report(); Assert.assertTrue(this.stream.toString().contains("tag1=value1")); String[] lines = this.stream.toString().split("\n"); Map<String, Set<String>> expected = new HashMap<>(); expectMetrics(expected, lines); Set<String> counterSubMetrics = new HashSet<>(); counterSubMetrics.add("count"); expected.put("com.linkedin.example.counter", counterSubMetrics); reporter.close(); }
@BeforeClass public void setUp() throws Exception { String contextName = CONTEXT_NAME + "_" + UUID.randomUUID().toString(); Reporter mockedReporter = Mockito.mock(Reporter.class); this.recordsProcessedCount = Mockito.mock(Counters.Counter.class); Mockito.when(mockedReporter.getCounter( contextName, MetricRegistry.name(RECORDS_PROCESSED, Measurements.COUNT.getName()))) .thenReturn(this.recordsProcessedCount); this.recordProcessRateCount = Mockito.mock(Counters.Counter.class); Mockito.when(mockedReporter.getCounter( contextName, MetricRegistry.name(RECORD_PROCESS_RATE, Measurements.COUNT.getName()))) .thenReturn(this.recordProcessRateCount); this.recordSizeDistributionCount = Mockito.mock(Counters.Counter.class); Mockito.when(mockedReporter.getCounter( contextName, MetricRegistry.name(RECORD_SIZE_DISTRIBUTION, Measurements.COUNT.getName()))) .thenReturn(this.recordSizeDistributionCount); this.totalDurationCount = Mockito.mock(Counters.Counter.class); Mockito.when(mockedReporter.getCounter( contextName, MetricRegistry.name(TOTAL_DURATION, Measurements.COUNT.getName()))) .thenReturn(this.totalDurationCount); this.queueSize = Mockito.mock(Counters.Counter.class); Mockito.when(mockedReporter.getCounter(contextName, QUEUE_SIZE)).thenReturn(this.queueSize); this.hadoopCounterReporter = HadoopCounterReporter.builder(mockedReporter) .convertRatesTo(TimeUnit.SECONDS) .convertDurationsTo(TimeUnit.SECONDS) .filter(MetricFilter.ALL) .build(MetricContext.builder(contextName).buildStrict()); }
@BeforeClass @SuppressWarnings("unchecked") public void setUp() { TaskInputOutputContext<Object, Object, Object, Object> mockContext = Mockito.mock(TaskInputOutputContext.class); this.recordsProcessedCount = Mockito.mock(Counter.class); Mockito.when(mockContext.getCounter( this.name, MetricRegistry.name(RECORDS_PROCESSED, Measurements.COUNT.getName()))) .thenReturn(this.recordsProcessedCount); this.recordProcessRateCount = Mockito.mock(Counter.class); Mockito.when(mockContext.getCounter( this.name, MetricRegistry.name(RECORD_PROCESS_RATE, Measurements.COUNT.getName()))) .thenReturn(this.recordProcessRateCount); this.recordSizeDistributionCount = Mockito.mock(Counter.class); Mockito.when(mockContext.getCounter( this.name, MetricRegistry.name(RECORD_SIZE_DISTRIBUTION, Measurements.COUNT.getName()))) .thenReturn(this.recordSizeDistributionCount); this.totalDurationCount = Mockito.mock(Counter.class); Mockito.when(mockContext.getCounter( this.name, MetricRegistry.name(TOTAL_DURATION, Measurements.COUNT.getName()))) .thenReturn(this.totalDurationCount); this.queueSize = Mockito.mock(Counter.class); Mockito.when(mockContext.getCounter(this.name, QUEUE_SIZE)).thenReturn(this.queueSize); this.hadoopCounterReporter = NewAPIHadoopCounterReporter.builder(mockContext) .convertRatesTo(TimeUnit.SECONDS) .convertDurationsTo(TimeUnit.SECONDS) .filter(MetricFilter.ALL) .build(MetricContext.builder(this.name).build()); }
@BeforeClass public void setUp() { this.context = MetricContext.builder(CONTEXT_NAME).build(); this.counter = ContextAwareMetricFactory.DEFAULT_CONTEXT_AWARE_COUNTER_FACTORY.newMetric( this.context, RECORDS_PROCESSED); this.meter = ContextAwareMetricFactory.DEFAULT_CONTEXT_AWARE_METER_FACTORY.newMetric( this.context, RECORD_PROCESS_RATE); this.histogram = ContextAwareMetricFactory.DEFAULT_CONTEXT_AWARE_HISTOGRAM_FACTORY.newMetric( this.context, RECORD_SIZE_DISTRIBUTION); this.timer = ContextAwareMetricFactory.DEFAULT_CONTEXT_AWARE_TIMER_FACTORY.newMetric( this.context, TOTAL_DURATION); }
@Test public void testWithTags() throws IOException { try ( MetricContext metricContext = MetricContext.builder(this.getClass().getCanonicalName() + ".testGraphiteReporter") .addTag(new Tag<String>("taskId", "task_testjob_123")) .addTag(new Tag<String>("forkBranchName", "fork_1")).build(); GraphiteReporter graphiteReporter = GraphiteReporter.Factory.newBuilder() .withGraphitePusher(graphitePusher) .withMetricContextName(CONTEXT_NAME) .build(new Properties());) { Counter counter = metricContext.counter(MetricRegistry.name(METRIC_PREFIX, COUNTER)); counter.inc(5l); graphiteReporter.report(new TreeMap<String, Gauge>(), metricContext.getCounters(), new TreeMap<String, Histogram>(), new TreeMap<String, Meter>(), new TreeMap<String, Timer>(), metricContext.getTagMap()); Assert.assertEquals(getMetricValue("task_testjob_123.fork_1." + METRIC_PREFIX, COUNTER, Measurements.COUNT), Long.toString(5l)); } }
@Test public void kafkaReporterContextTest() throws IOException { Tag<?> tag1 = new Tag<>("tag1", "value1"); MetricContext context = MetricContext.builder("context").addTag(tag1).build(); Counter counter = context.counter("com.linkedin.example.counter"); MockKafkaPusher pusher = new MockKafkaPusher(); KafkaReporter kafkaReporter = getBuilderFromContext(pusher).build("localhost:0000", "topic", new Properties()); counter.inc(); kafkaReporter.report(context); try { Thread.sleep(1000); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } MetricReport metricReport = nextReport(pusher.messageIterator()); Assert.assertEquals(3, metricReport.getTags().size()); Assert.assertTrue(metricReport.getTags().containsKey(tag1.getKey())); Assert.assertEquals(metricReport.getTags().get(tag1.getKey()), tag1.getValue().toString()); }
@Test public void testSimpleEvent() throws IOException { try ( MetricContext metricContext = MetricContext.builder(this.getClass().getCanonicalName() + ".testInfluxDBReporter1").build(); InfluxDBEventReporter influxEventReporter = getBuilder(metricContext).build();) { Map<String, String> metadata = Maps.newHashMap(); metadata.put(JobEvent.METADATA_JOB_ID, "job1"); metadata.put(TaskEvent.METADATA_TASK_ID, "task1"); metricContext.submitEvent(GobblinTrackingEvent.newBuilder() .setName(JobEvent.TASKS_SUBMITTED) .setNamespace(NAMESPACE) .setMetadata(metadata).build()); try { Thread.sleep(100); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } influxEventReporter.report(); try { Thread.sleep(100); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } TimestampedValue retrievedEvent = influxDB.getMetric( "gobblin.metrics.job1.task1.events.TasksSubmitted"); Assert.assertEquals(retrievedEvent.getValue(), "0.0"); Assert.assertTrue(retrievedEvent.getTimestamp() <= (System.currentTimeMillis())); } }
@Test public void testWithTags() throws IOException { try ( MetricContext metricContext = MetricContext.builder(this.getClass().getCanonicalName() + ".testGraphiteReporter") .addTag(new Tag<String>("taskId", "task_testjob_123")) .addTag(new Tag<String>("forkBranchName", "fork_1")).build(); InfluxDBReporter influxDBReporter = InfluxDBReporter.Factory.newBuilder() .withInfluxDBPusher(influxDBPusher) .withMetricContextName(CONTEXT_NAME) .build(new Properties());) { Counter counter = metricContext.counter(MetricRegistry.name(METRIC_PREFIX, COUNTER)); counter.inc(5l); influxDBReporter.report(new TreeMap<String, Gauge>(), metricContext.getCounters(), new TreeMap<String, Histogram>(), new TreeMap<String, Meter>(), new TreeMap<String, Timer>(), metricContext.getTagMap()); //InfluxDB converts all values to float64 internally Assert.assertEquals(getMetricValue("task_testjob_123.fork_1." + METRIC_PREFIX, COUNTER, Measurements.COUNT), Float.toString(5f)); } }
@Test public void kafkaReporterTagsTest() throws IOException { MetricContext metricContext = MetricContext.builder(this.getClass().getCanonicalName() + ".kafkaReporterTagsTest").build(); Counter counter = metricContext.counter("com.linkedin.example.counter"); Tag<?> tag1 = new Tag<>("tag1", "value1"); Tag<?> tag2 = new Tag<>("tag2", 2); MockKafkaPusher pusher = new MockKafkaPusher(); KafkaReporter kafkaReporter = getBuilder(pusher).withTags(Lists.newArrayList(tag1, tag2)).build("localhost:0000", "topic", new Properties()); counter.inc(); kafkaReporter.report(metricContext); try { Thread.sleep(1000); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } MetricReport metricReport = nextReport(pusher.messageIterator()); Assert.assertEquals(4, metricReport.getTags().size()); Assert.assertTrue(metricReport.getTags().containsKey(tag1.getKey())); Assert.assertEquals(metricReport.getTags().get(tag1.getKey()), tag1.getValue().toString()); Assert.assertTrue(metricReport.getTags().containsKey(tag2.getKey())); Assert.assertEquals(metricReport.getTags().get(tag2.getKey()), tag2.getValue().toString()); }
@Test public void testSimpleEvent() throws IOException { try ( MetricContext metricContext = MetricContext.builder(this.getClass().getCanonicalName() + ".testGraphiteReporter1").build(); GraphiteEventReporter graphiteEventReporter = getBuilder(metricContext).withEmitValueAsKey(false).build();) { Map<String, String> metadata = Maps.newHashMap(); metadata.put(JobEvent.METADATA_JOB_ID, "job1"); metadata.put(TaskEvent.METADATA_TASK_ID, "task1"); metricContext.submitEvent(GobblinTrackingEvent.newBuilder() .setName(JobEvent.TASKS_SUBMITTED) .setNamespace(NAMESPACE) .setMetadata(metadata).build()); try { Thread.sleep(100); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } graphiteEventReporter.report(); try { Thread.sleep(100); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } TimestampedValue retrievedEvent = graphiteSender.getMetric("gobblin.metrics.job1.task1.events.TasksSubmitted"); Assert.assertEquals(retrievedEvent.getValue(), "0"); Assert.assertTrue(retrievedEvent.getTimestamp() <= (System.currentTimeMillis() / 1000l)); } }
@Test public void testKafkaEventReporter() throws IOException { MetricContext context = MetricContext.builder("context").build();
try ( MetricContext metricContext = MetricContext.builder(this.getClass().getCanonicalName() + ".testInfluxDBReporter2").build();
@Test public void testKafkaEventReporter() throws IOException { MetricContext context = MetricContext.builder("context").build();