private void updateGauge(Iterable<MetricResult<GaugeResult>> gauges) { for (MetricResult<GaugeResult> metricResult : gauges) { String flinkMetricName = getFlinkMetricNameString(metricResult); GaugeResult update = metricResult.getAttempted(); // update flink metric FlinkGauge gauge = flinkGaugeCache.get(flinkMetricName); if (gauge == null) { gauge = runtimeContext.getMetricGroup().gauge(flinkMetricName, new FlinkGauge(update)); flinkGaugeCache.put(flinkMetricName, gauge); } else { gauge.update(update); } } }
@Override protected void describeMismatchSafely(MetricResult<T> item, Description mismatchDescription) { mismatchDescription.appendText("MetricResult{"); final T metricValue = isCommitted ? item.getCommitted() : item.getAttempted(); describeMetricsResultMembersMismatch(item, mismatchDescription, namespace, name, step); if (!Objects.equals(value, metricValue)) { mismatchDescription .appendText(String.format("%s: ", metricState)) .appendValue(value) .appendText(" != ") .appendValue(metricValue); } mismatchDescription.appendText("}"); } };
@Override public void writeMetrics(MetricQueryResults metricQueryResults) throws Exception { counterValue = metricQueryResults.getCounters().iterator().hasNext() ? metricQueryResults.getCounters().iterator().next().getAttempted() : 0L; } }
private void updateGauge(Iterable<MetricResult<GaugeResult>> gauges) { for (MetricResult<GaugeResult> metricResult : gauges) { String flinkMetricName = getFlinkMetricNameString(GAUGE_PREFIX, metricResult); GaugeResult update = metricResult.getAttempted(); // update flink metric FlinkGauge gauge = flinkGaugeCache.get(flinkMetricName); if (gauge == null) { gauge = runtimeContext.getMetricGroup().gauge(flinkMetricName, new FlinkGauge(update)); flinkGaugeCache.put(flinkMetricName, gauge); } else { gauge.update(update); } } }
private void updateDistributions(Iterable<MetricResult<DistributionResult>> distributions) { for (MetricResult<DistributionResult> metricResult : distributions) { String flinkMetricName = getFlinkMetricNameString(metricResult); DistributionResult update = metricResult.getAttempted(); // update flink metric FlinkDistributionGauge gauge = flinkDistributionGaugeCache.get(flinkMetricName); if (gauge == null) { gauge = runtimeContext .getMetricGroup() .gauge(flinkMetricName, new FlinkDistributionGauge(update)); flinkDistributionGaugeCache.put(flinkMetricName, gauge); } else { gauge.update(update); } } }
private void updateDistributions(Iterable<MetricResult<DistributionResult>> distributions) { for (MetricResult<DistributionResult> metricResult : distributions) { String flinkMetricName = getFlinkMetricNameString(DISTRIBUTION_PREFIX, metricResult); DistributionResult update = metricResult.getAttempted(); // update flink metric FlinkDistributionGauge gauge = flinkDistributionGaugeCache.get(flinkMetricName); if (gauge == null) { gauge = runtimeContext .getMetricGroup() .gauge(flinkMetricName, new FlinkDistributionGauge(update)); flinkDistributionGaugeCache.put(flinkMetricName, gauge); } else { gauge.update(update); } } }
private void updateCounters(Iterable<MetricResult<Long>> counters) { for (MetricResult<Long> metricResult : counters) { String flinkMetricName = getFlinkMetricNameString(COUNTER_PREFIX, metricResult); Long update = metricResult.getAttempted(); // update flink metric Counter counter = flinkCounterCache.computeIfAbsent( flinkMetricName, n -> runtimeContext.getMetricGroup().counter(n)); counter.dec(counter.getCount()); counter.inc(update); } }
private void updateCounters(Iterable<MetricResult<Long>> counters) { for (MetricResult<Long> metricResult : counters) { String flinkMetricName = getFlinkMetricNameString(metricResult); Long update = metricResult.getAttempted(); // update flink metric Counter counter = flinkCounterCache.computeIfAbsent( flinkMetricName, n -> runtimeContext.getMetricGroup().counter(n)); counter.dec(counter.getCount()); counter.inc(update); } }
Map<String, ?> renderAll() { Map<String, Object> metrics = new HashMap<>(); MetricResults metricResults = asAttemptedOnlyMetricResults(MetricsAccumulator.getInstance().value()); MetricQueryResults metricQueryResults = metricResults.queryMetrics(MetricsFilter.builder().build()); for (MetricResult<Long> metricResult : metricQueryResults.getCounters()) { metrics.put(renderName(metricResult), metricResult.getAttempted()); } for (MetricResult<DistributionResult> metricResult : metricQueryResults.getDistributions()) { DistributionResult result = metricResult.getAttempted(); metrics.put(renderName(metricResult) + ".count", result.getCount()); metrics.put(renderName(metricResult) + ".sum", result.getSum()); metrics.put(renderName(metricResult) + ".min", result.getMin()); metrics.put(renderName(metricResult) + ".max", result.getMax()); metrics.put(renderName(metricResult) + ".mean", result.getMean()); } for (MetricResult<GaugeResult> metricResult : metricQueryResults.getGauges()) { metrics.put(renderName(metricResult), metricResult.getAttempted().getValue()); } return metrics; }
/** * Return the current value for a long counter, or a default value if can't be retrieved. Note * this uses only attempted metrics because some runners don't support committed metrics. */ private long getDistributionMetric(String name, DistributionType distType, long defaultValue) { MetricQueryResults metrics = result .metrics() .queryMetrics( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(namespace, name)) .build()); Iterable<MetricResult<DistributionResult>> distributions = metrics.getDistributions(); checkIfMetricResultIsUnique(name, distributions); try { MetricResult<DistributionResult> distributionResult = distributions.iterator().next(); switch (distType) { case MIN: return distributionResult.getAttempted().getMin(); case MAX: return distributionResult.getAttempted().getMax(); default: return defaultValue; } } catch (NoSuchElementException e) { LOG.error("Failed to get distribution metric {} for namespace {}", name, namespace); } return defaultValue; }
/** * Return the current value for a long counter, or a default value if can't be retrieved. Note * this uses only attempted metrics because some runners don't support committed metrics. */ public long getCounterMetric(String name, long defaultValue) { MetricQueryResults metrics = result .metrics() .queryMetrics( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(namespace, name)) .build()); Iterable<MetricResult<Long>> counters = metrics.getCounters(); checkIfMetricResultIsUnique(name, counters); try { MetricResult<Long> metricResult = counters.iterator().next(); return metricResult.getAttempted(); } catch (NoSuchElementException e) { LOG.error("Failed to get metric {}, from namespace {}", name, namespace); } return defaultValue; }
@Override protected void describeMismatchSafely( MetricResult<DistributionResult> item, Description mismatchDescription) { mismatchDescription.appendText("MetricResult{"); describeMetricsResultMembersMismatch(item, mismatchDescription, namespace, name, step); DistributionResult metricValue = isCommitted ? item.getCommitted() : item.getAttempted(); if (!Objects.equals(min, metricValue.getMin())) { mismatchDescription .appendText(String.format("%sMin: ", metricState)) .appendValue(min) .appendText(" != ") .appendValue(metricValue.getMin()); } if (!Objects.equals(max, metricValue.getMax())) { mismatchDescription .appendText(String.format("%sMax: ", metricState)) .appendValue(max) .appendText(" != ") .appendValue(metricValue.getMax()); } mismatchDescription.appendText("}"); } };
.getCounters(); for (MetricResult<Long> counter : successCounterResults) { if (counter.getAttempted() > 0) { successfulAssertions++;
@Override protected boolean matchesSafely(MetricResult<DistributionResult> item) { DistributionResult metricValue = isCommitted ? item.getCommitted() : item.getAttempted(); return Objects.equals(namespace, item.getName().getNamespace()) && Objects.equals(name, item.getName().getName()) && item.getStep().contains(step) && Objects.equals(min, metricValue.getMin()) && Objects.equals(max, metricValue.getMax()); }
@Override protected boolean matchesSafely(MetricResult<T> item) { final T metricValue = isCommitted ? item.getCommitted() : item.getAttempted(); return Objects.equals(namespace, item.getName().getNamespace()) && Objects.equals(name, item.getName().getName()) && item.getStep().contains(step) && metricResultsEqual(value, metricValue); }
public static void queryMetrics(PipelineResult result) { MetricQueryResults metrics = result.metrics().queryMetrics( MetricsFilter.builder().addNameFilter(MetricNameFilter.inNamespace("PollingExample")).build()); Iterable<MetricResult<Long>> counters = metrics.getCounters(); for (MetricResult<Long> counter : counters) { System.out.println(counter.getName().name() + " : " + counter.getAttempted() + " " + Instant.now()); } }
private static Enumerable<Object> count(PipelineOptions options, BeamRelNode node) { Pipeline pipeline = Pipeline.create(options); BeamSqlRelUtils.toPCollection(pipeline, node).apply(ParDo.of(new RowCounter())); PipelineResult result = pipeline.run(); long count = 0; if (!containsUnboundedPCollection(pipeline)) { result.waitUntilFinish(); MetricQueryResults metrics = result .metrics() .queryMetrics( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(BeamEnumerableConverter.class, "rows")) .build()); Iterator<MetricResult<Long>> iterator = metrics.getCounters().iterator(); if (iterator.hasNext()) { count = iterator.next().getAttempted(); } } return Linq4j.singletonEnumerable(count); }
/** * Create a feature row from Metrics. * The granularity is unrelated to the metrics themselves, and simply indicates the * granularity at which to store and overwrite the metrics downstream. * * @param counter * @param jobName * @param granularity * @return */ public static FeatureRow makeFeatureRow( MetricResult<Long> counter, String jobName, Granularity.Enum granularity) { String jobId = jobName; String namespace = counter.getName().getNamespace(); String step = counter.getStep(); String name = counter.getName().getName(); Long attempted = counter.getAttempted(); String entityId = String.join(":", new String[] {jobId, namespace, step, name}); return FeatureRow.newBuilder() .setEntityName(METRICS_ENTITY_NAME) .setEntityKey(entityId) .setGranularity(granularity) .setEventTimestamp(DateUtil.toTimestamp(DateTime.now())) .addFeatures(Features.of(METRICS_FEATURE_JOB_ID, Values.ofString(jobId))) .addFeatures(Features.of(METRICS_FEATURE_NAMESPACE, Values.ofString(namespace))) .addFeatures(Features.of(METRICS_FEATURE_STEP, Values.ofString(step))) .addFeatures(Features.of(METRICS_FEATURE_NAME, Values.ofString(name))) .addFeatures(Features.of(METRICS_FEATURE_ATTEMPTED, Values.ofInt64(attempted))) .build(); }
commitsEnqueuedMetrics.getCounters().iterator().next().getAttempted(), greaterThan(0L));