private void computePartitionMetrics(@NonNull final TopicPartition topicPartition, final long leaderOffset, @NonNull final KafkaRunState nextRunState, @NonNull final DataFeedMetrics topicMetrics, @NonNull final Long inputCount, @NonNull final MessageCounters counter) { final Optional<Long> nextPartitionState = nextRunState.getPartitionOffset(topicPartition.partition()); final Map<String, String> tags = new HashMap<>(); tags.put(PARTITION_TAG, Integer.toString(topicPartition.partition())); if (nextPartitionState.isPresent()) { final long current = nextPartitionState.get(); counter.add(leaderOffset, current, inputCount); topicMetrics.createLongMetric(DataFeedMetricNames.AVAILABLE_ROWCOUNT, leaderOffset, tags); topicMetrics.createLongMetric(DataFeedMetricNames.CURRENT_STATUS, current, tags); topicMetrics.createLongMetric(DataFeedMetricNames.INPUT_ROWCOUNT, inputCount, tags); } else { topicMetrics.createLongMetric(DataFeedMetricNames.AVAILABLE_ROWCOUNT, leaderOffset, tags); topicMetrics.createLongMetric(DataFeedMetricNames.CURRENT_STATUS, leaderOffset, tags); topicMetrics.createLongMetric(DataFeedMetricNames.INPUT_ROWCOUNT, 0, tags); } }
); topicMetrics.createLongMetric(DataFeedMetricNames.ROWCOUNT_BEHIND, counter.getTotalAvailable() - counter.getTotalCurrent(), totalTags); topicMetrics.createLongMetric(DataFeedMetricNames.INPUT_ROWCOUNT, counter.getTotalInput(), totalTags); if (this.chargebackCalculator.isPresent()) { this.chargebackCalculator.get().addCost(
private void logWriteMetrics(final Optional<JavaRDD<WriteStatus>> writesStatuses) { if (writesStatuses.isPresent() && this.dataFeedMetrics.isPresent()) { final LongAccumulator totalCount = writesStatuses.get().rdd().sparkContext().longAccumulator(); final LongAccumulator errorCount = writesStatuses.get().rdd().sparkContext().longAccumulator(); writesStatuses.get().foreach(writeStatus -> { errorCount.add(writeStatus.getFailedRecords().size()); totalCount.add(writeStatus.getTotalRecords()); }); this.dataFeedMetrics.get().createLongMetric(DataFeedMetricNames.ERROR_ROWCOUNT, errorCount.value(), this.dataFeedMetricsTags); this.dataFeedMetrics.get().createLongMetric(DataFeedMetricNames.OUTPUT_ROWCOUNT, totalCount.value() - errorCount.value(), this.dataFeedMetricsTags); } }
tags.put(TABLE_NAME_TAG, this.conf.getKeyspace() + StringTypes.UNDERSCORE + this.conf.getTableName()); this.tableMetrics.get() .createLongMetric(DataFeedMetricNames.OUTPUT_ROWCOUNT, payloadWrapper.getCount(), tags); .createLongMetric(DataFeedMetricNames.OUTPUT_BYTE_SIZE, cassandraPayloadRddSizeEstimator.estimateTotalSize(payloadWrapper), tags);
String.valueOf(TimeUnit.SECONDS.toMillis(timerMetric.getStartTime().getEpochSecond()))); this.dataFeedMetrics.createLongMetric(DataFeedMetricNames.RESULT, successful.get() ? DataFeedMetricNames.RESULT_SUCCESS : DataFeedMetricNames.RESULT_FAILURE, Collections.emptyMap());
final RDDWrapper<String> dataWrapper = new RDDWrapper<>(dataToWrite); final long totalRows = dataWrapper.getCount(); this.dataFeedMetrics.get().createLongMetric(DataFeedMetricNames.OUTPUT_ROWCOUNT, totalRows, tags);
tags.put(TABLE_NAME_TAG, this.conf.getKeyspace() + StringTypes.UNDERSCORE + this.conf.getTableName()); this.tableMetrics.get() .createLongMetric(DataFeedMetricNames.OUTPUT_ROWCOUNT, payloadWrapper.getCount(), tags);
@Test public void testCreateTopicLongMetric() { Metric topicMetric1 = dataFeedMetrics.createLongMetric(DataFeedMetricNames.OUTPUT_ROWCOUNT, 100l, Maps.newHashMap()); Metric topicMetric2 = dataFeedMetrics.createLongMetric(DataFeedMetricNames.SIZE_BYTES, 1028l, Maps.newHashMap()); assertEquals("output_rowcount", topicMetric1.getMetricName()); assertEquals("size_bytes", topicMetric2.getMetricName()); assertEquals(new Long(100), topicMetric1.getMetricValue()); assertEquals(new Long(1028), topicMetric2.getMetricValue()); assertTrue(topicMetric1.getTags().containsKey("job")); assertEquals(topicMetric1.getTags().get("job"), "test-job"); assertTrue(topicMetric1.getTags().containsKey("datafeed")); assertEquals(topicMetric1.getTags().get("datafeed"), "test-topic"); assertFalse(topicMetric1.getTags().containsKey("hello")); topicMetric1.addTag("hello", "world"); assertTrue(topicMetric1.getTags().containsKey("hello")); assertTrue(dataFeedMetrics.getMetricSet().contains(topicMetric1)); assertTrue(dataFeedMetrics.getMetricSet().contains(topicMetric2)); }