/** * Return the current value for a long counter, or a default value if can't be retrieved. Note * this uses only attempted metrics because some runners don't support committed metrics. */ public long getCounterMetric(String name, long defaultValue) { MetricQueryResults metrics = result .metrics() .queryMetrics( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(namespace, name)) .build()); Iterable<MetricResult<Long>> counters = metrics.getCounters(); checkIfMetricResultIsUnique(name, counters); try { MetricResult<Long> metricResult = counters.iterator().next(); return metricResult.getAttempted(); } catch (NoSuchElementException e) { LOG.error("Failed to get metric {}, from namespace {}", name, namespace); } return defaultValue; }
.queryMetrics( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(PAssert.class, PAssert.SUCCESS_COUNTER)) .build()) .getCounters();
/** * Return the current value for a long counter, or a default value if can't be retrieved. Note * this uses only attempted metrics because some runners don't support committed metrics. */ private long getDistributionMetric(String name, DistributionType distType, long defaultValue) { MetricQueryResults metrics = result .metrics() .queryMetrics( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(namespace, name)) .build()); Iterable<MetricResult<DistributionResult>> distributions = metrics.getDistributions(); checkIfMetricResultIsUnique(name, distributions); try { MetricResult<DistributionResult> distributionResult = distributions.iterator().next(); switch (distType) { case MIN: return distributionResult.getAttempted().getMin(); case MAX: return distributionResult.getAttempted().getMax(); default: return defaultValue; } } catch (NoSuchElementException e) { LOG.error("Failed to get distribution metric {} for namespace {}", name, namespace); } return defaultValue; }
private static Enumerable<Object> count(PipelineOptions options, BeamRelNode node) { Pipeline pipeline = Pipeline.create(options); BeamSqlRelUtils.toPCollection(pipeline, node).apply(ParDo.of(new RowCounter())); PipelineResult result = pipeline.run(); long count = 0; if (!containsUnboundedPCollection(pipeline)) { result.waitUntilFinish(); MetricQueryResults metrics = result .metrics() .queryMetrics( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(BeamEnumerableConverter.class, "rows")) .build()); Iterator<MetricResult<Long>> iterator = metrics.getCounters().iterator(); if (iterator.hasNext()) { count = iterator.next().getAttempted(); } } return Linq4j.singletonEnumerable(count); }
@Test public void testMatchStringNamespaceFilters() { // MetricsFilter with a String-namespace + name filter. Without step filter. // Successful match. assertTrue( MetricFiltering.matches( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named("myNamespace", "myMetricName")) .build(), MetricKey.create("anyStep", MetricName.named("myNamespace", "myMetricName")))); // Unsuccessful match. assertFalse( MetricFiltering.matches( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named("myOtherNamespace", "myMetricName")) .build(), MetricKey.create("anyStep", MetricName.named("myNamespace", "myMetricname")))); }
@Test public void testMatchClassNamespaceFilters() { // MetricsFilter with a Class-namespace + name filter. Without step filter. // Successful match. assertTrue( MetricFiltering.matches( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(MetricFilteringTest.class, "myMetricName")) .build(), MetricKey.create( "anyStep", MetricName.named(MetricFilteringTest.class, "myMetricName")))); // Unsuccessful match. assertFalse( MetricFiltering.matches( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(MetricFilteringTest.class, "myMetricName")) .build(), MetricKey.create("anyStep", MetricName.named(MetricFiltering.class, "myMetricName")))); }
@Test public void testMatchStepNameFilters() { // MetricsFilter with a Class-namespace + name filter + step filter. // Successful match. assertTrue( MetricFiltering.matches( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(MetricFilteringTest.class, "myMetricName")) .addStep("myStep") .build(), MetricKey.create( "myStep", MetricName.named(MetricFilteringTest.class, "myMetricName")))); // Unsuccessful match. assertFalse( MetricFiltering.matches( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(MetricFilteringTest.class, "myMetricName")) .addStep("myOtherStep") .build(), MetricKey.create( "myStep", MetricName.named(MetricFilteringTest.class, "myMetricName")))); }
@Test public void testMatchCompositeStepNameFilters() { // MetricsFilter with a Class-namespace + name filter + step filter. // Successful match. assertTrue( MetricFiltering.matches( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(MetricFilteringTest.class, "myMetricName")) .addStep("myStep") .build(), MetricKey.create( "myBigStep/myStep", MetricName.named(MetricFilteringTest.class, "myMetricName")))); // Unsuccessful match. assertFalse( MetricFiltering.matches( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(MetricFilteringTest.class, "myMetricName")) .addStep("myOtherStep") .build(), MetricKey.create( "myOtherStepNoMatch/myStep", MetricName.named(MetricFilteringTest.class, "myMetricName")))); }
@Test @Category({NeedsRunner.class, UsesAttemptedMetrics.class, UsesCounterMetrics.class}) public void testBoundedSourceMetrics() { long numElements = 1000; pipeline.apply(GenerateSequence.from(0).to(numElements)); PipelineResult pipelineResult = pipeline.run(); MetricQueryResults metrics = pipelineResult .metrics() .queryMetrics( MetricsFilter.builder() .addNameFilter( MetricNameFilter.named( ELEMENTS_READ.getNamespace(), ELEMENTS_READ.getName())) .build()); assertThat( metrics.getCounters(), hasItem( attemptedMetricsResult( ELEMENTS_READ.getNamespace(), ELEMENTS_READ.getName(), "Read(BoundedCountingSource)", 1000L))); }
@Test @Category({NeedsRunner.class, UsesAttemptedMetrics.class, UsesCounterMetrics.class}) public void testUnboundedSourceMetrics() { long numElements = 1000; // Use withMaxReadTime to force unbounded mode. pipeline.apply( GenerateSequence.from(0).to(numElements).withMaxReadTime(Duration.standardDays(1))); PipelineResult pipelineResult = pipeline.run(); MetricQueryResults metrics = pipelineResult .metrics() .queryMetrics( MetricsFilter.builder() .addNameFilter( MetricNameFilter.named( ELEMENTS_READ.getNamespace(), ELEMENTS_READ.getName())) .build()); assertThat( metrics.getCounters(), hasItem( attemptedMetricsResult( ELEMENTS_READ.getNamespace(), ELEMENTS_READ.getName(), "Read(UnboundedCountingSource)", 1000L))); } }
MetricsFilter.builder() .addNameFilter( MetricNameFilter.named( backlogElementsOfSplit.getNamespace(), backlogElementsOfSplit.getName())) MetricsFilter.builder() .addNameFilter( MetricNameFilter.named( backlogBytesOfSplit.getNamespace(), backlogBytesOfSplit.getName())) .build()); MetricsFilter.builder() .addNameFilter( MetricNameFilter.named( KafkaUnboundedReader.METRIC_NAMESPACE, KafkaUnboundedReader.CHECKPOINT_MARK_COMMITS_ENQUEUED_METRIC))