public static void update(String name, long value) { Metrics.distribution(FeastMetrics.FEAST_NAMESPACE, name).update(value); }
@ProcessElement public void processElement(ProcessContext c) throws IOException, GeneralSecurityException { Metrics.counter(RetrieveReads.class, "Initialized Shard Count").inc(); Stopwatch stopWatch = Stopwatch.createStarted(); Iterator<StreamReadsResponse> iter = ReadStreamIterator.enforceShardBoundary(auth, c.element(), shardBoundary, fields); while (iter.hasNext()) { StreamReadsResponse readResponse = iter.next(); c.output(readResponse.getAlignmentsList()); } stopWatch.stop(); Metrics.distribution(RetrieveReads.class, "Shard Processing Time (sec)") .update(stopWatch.elapsed(TimeUnit.SECONDS)); Metrics.counter(RetrieveReads.class, "Finished Shard Count").inc(); } }
@ProcessElement public void processElement(ProcessContext c) throws IOException, GeneralSecurityException { Metrics.counter(RetrieveReads.class, "Initialized Shard Count").inc(); Stopwatch stopWatch = Stopwatch.createStarted(); Iterator<StreamReadsResponse> iter = ReadStreamIterator.enforceShardBoundary(auth, c.element(), shardBoundary, fields); while (iter.hasNext()) { StreamReadsResponse readResponse = iter.next(); c.output(readResponse.getAlignmentsList()); } stopWatch.stop(); Metrics.distribution(RetrieveReads.class, "Shard Processing Time (sec)") .update(stopWatch.elapsed(TimeUnit.SECONDS)); Metrics.counter(RetrieveReads.class, "Finished Shard Count").inc(); } }
@ProcessElement public void processElement(ProcessContext c) throws IOException, GeneralSecurityException, InterruptedException { Metrics.counter(RetrieveVariants.class, "Initialized Shard Count").inc(); Stopwatch stopWatch = Stopwatch.createStarted(); Iterator<StreamVariantsResponse> iter = VariantStreamIterator.enforceShardBoundary(auth, c.element(), shardBoundary, fields); while (iter.hasNext()) { StreamVariantsResponse variantResponse = iter.next(); c.output(variantResponse.getVariantsList()); } stopWatch.stop(); Metrics.distribution(RetrieveVariants.class, "Shard Processing Time (sec)") .update(stopWatch.elapsed(TimeUnit.SECONDS)); Metrics.counter(RetrieveVariants.class, "Finished Shard Count").inc(); stats.addValue(stopWatch.elapsed(TimeUnit.SECONDS)); LOG.info("Shard Duration in Seconds - Min: " + stats.getMin() + " Max: " + stats.getMax() + " Avg: " + stats.getMean() + " StdDev: " + stats.getStandardDeviation()); } }
@Test public void testDistributionWithEmptyNamespace() { thrown.expect(IllegalArgumentException.class); Metrics.distribution("", NAME); }
@Test public void testDistributionWithEmptyName() { thrown.expect(IllegalArgumentException.class); Metrics.distribution(NS, ""); }
@ProcessElement public void processElement(ProcessContext c) throws IOException, GeneralSecurityException, InterruptedException { Metrics.counter(RetrieveVariants.class, "Initialized Shard Count").inc(); Stopwatch stopWatch = Stopwatch.createStarted(); Iterator<StreamVariantsResponse> iter = VariantStreamIterator.enforceShardBoundary(auth, c.element(), shardBoundary, fields); while (iter.hasNext()) { StreamVariantsResponse variantResponse = iter.next(); c.output(variantResponse.getVariantsList()); } stopWatch.stop(); Metrics.distribution(RetrieveVariants.class, "Shard Processing Time (sec)") .update(stopWatch.elapsed(TimeUnit.SECONDS)); Metrics.counter(RetrieveVariants.class, "Finished Shard Count").inc(); stats.addValue(stopWatch.elapsed(TimeUnit.SECONDS)); LOG.info("Shard Duration in Seconds - Min: " + stats.getMin() + " Max: " + stats.getMax() + " Avg: " + stats.getMean() + " StdDev: " + stats.getStandardDeviation()); } }
@FinishBundle public void finishBundle(DoFn<Read, String>.FinishBundleContext c) throws IOException { bw.close(); Metrics.distribution(WriteBAMFn.class, "Maximum Write Shard Processing Time (sec)") .update(stopWatch.elapsed(TimeUnit.SECONDS)); LOG.info("Finished writing " + shardContig); Metrics.counter(WriteBAMFn.class, "Finished Write Shard Count").inc(); final long bytesWritten = ts.getBytesWrittenExceptingTruncation(); LOG.info("Wrote " + readCount + " reads, " + unmappedReadCount + " unmapped, into " + shardName + (hadOutOfOrder ? "ignored out of order" : "") + ", wrote " + bytesWritten + " bytes"); Metrics.counter(WriteBAMFn.class, "Written reads").inc(readCount); Metrics.counter(WriteBAMFn.class, "Written unmapped reads").inc(unmappedReadCount); final long totalReadCount = (long)readCount + (long)unmappedReadCount; Metrics.distribution(WriteBAMFn.class, "Maximum Reads Per Shard").update(totalReadCount); c.output(shardName, window.maxTimestamp(), window); c.output(SEQUENCE_SHARD_SIZES_TAG, KV.of(sequenceIndex, bytesWritten), window.maxTimestamp(), window); }
@FinishBundle public void finishBundle(DoFn<Read, String>.FinishBundleContext c) throws IOException { bw.close(); Metrics.distribution(WriteBAMFn.class, "Maximum Write Shard Processing Time (sec)") .update(stopWatch.elapsed(TimeUnit.SECONDS)); LOG.info("Finished writing " + shardContig); Metrics.counter(WriteBAMFn.class, "Finished Write Shard Count").inc(); final long bytesWritten = ts.getBytesWrittenExceptingTruncation(); LOG.info("Wrote " + readCount + " reads, " + unmappedReadCount + " unmapped, into " + shardName + (hadOutOfOrder ? "ignored out of order" : "") + ", wrote " + bytesWritten + " bytes"); Metrics.counter(WriteBAMFn.class, "Written reads").inc(readCount); Metrics.counter(WriteBAMFn.class, "Written unmapped reads").inc(unmappedReadCount); final long totalReadCount = (long)readCount + (long)unmappedReadCount; Metrics.distribution(WriteBAMFn.class, "Maximum Reads Per Shard").update(totalReadCount); c.output(shardName, window.maxTimestamp(), window); c.output(SEQUENCE_SHARD_SIZES_TAG, KV.of(sequenceIndex, bytesWritten), window.maxTimestamp(), window); }
@Test public void testDistributionToCell() { MetricsContainer mockContainer = Mockito.mock(MetricsContainer.class); Distribution mockDistribution = Mockito.mock(Distribution.class); when(mockContainer.getDistribution(METRIC_NAME)).thenReturn(mockDistribution); Distribution distribution = Metrics.distribution(NS, NAME); MetricsEnvironment.setCurrentContainer(mockContainer); distribution.update(5L); verify(mockDistribution).update(5L); distribution.update(36L); distribution.update(1L); verify(mockDistribution).update(36L); verify(mockDistribution).update(1L); }
@Test public void testDistributionWithoutContainer() { assertNull(MetricsEnvironment.getCurrentContainer()); // Should not fail even though there is no metrics container. Metrics.distribution(NS, NAME).update(5L); }
noCoordinateReads + " no coordinate reads, " + skippedReads + ", skipped reads"); stopWatch.stop(); Metrics.distribution(WriteBAIFn.class, "Indexing Shard Processing Time (sec)").update( stopWatch.elapsed(TimeUnit.SECONDS)); Metrics.counter(WriteBAIFn.class, "Finished Indexing Shard Count").inc(); Metrics.counter(WriteBAIFn.class, "Indexed reads").inc(processedReads); Metrics.counter(WriteBAIFn.class, "Indexed no coordinate reads").inc(noCoordinateReads); Metrics.distribution(WriteBAIFn.class, "Reads Per Indexing Shard").update(processedReads);
noCoordinateReads + " no coordinate reads, " + skippedReads + ", skipped reads"); stopWatch.stop(); Metrics.distribution(WriteBAIFn.class, "Indexing Shard Processing Time (sec)").update( stopWatch.elapsed(TimeUnit.SECONDS)); Metrics.counter(WriteBAIFn.class, "Finished Indexing Shard Count").inc(); Metrics.counter(WriteBAIFn.class, "Indexed reads").inc(processedReads); Metrics.counter(WriteBAIFn.class, "Indexed no coordinate reads").inc(noCoordinateReads); Metrics.distribution(WriteBAIFn.class, "Reads Per Indexing Shard").update(processedReads);
@SuppressWarnings("unused") @ProcessElement public void processElement(ProcessContext c) { Distribution values = Metrics.distribution(MetricsTest.class, "input"); count.inc(); values.update(c.element()); c.output(c.element()); c.output(c.element()); }
@SuppressWarnings("unused") @ProcessElement public void processElement(ProcessContext c) { Distribution values = Metrics.distribution(MetricsTest.class, "input"); Gauge gauge = Metrics.gauge(MetricsTest.class, "my-gauge"); Integer element = c.element(); count.inc(); values.update(element); gauge.set(12L); c.output(element); c.output(output2, element); } })
ParDo.of( new DoFn<Integer, Integer>() { Distribution bundleDist = Metrics.distribution(MetricsTest.class, "bundle");