private void refreshLagMetrics() { for (Map.Entry<SystemStreamPartition, Long> e : nextOffsets.entrySet()) { SystemStreamPartition ssp = e.getKey(); Long offset = e.getValue(); TopicAndPartition tp = new TopicAndPartition(ssp.getStream(), ssp.getPartition().getPartitionId()); Long lag = latestLags.get(ssp); LOG.trace("Latest offset of {} is {}; lag = {}", ssp, offset, lag); if (lag != null && offset != null && lag >= 0) { long streamEndOffset = offset.longValue() + lag.longValue(); // update the metrics kafkaConsumerMetrics.setHighWatermarkValue(tp, streamEndOffset); kafkaConsumerMetrics.setLagValue(tp, lag.longValue()); } } }
private void refreshLagMetrics() { for (Map.Entry<SystemStreamPartition, Long> e : nextOffsets.entrySet()) { SystemStreamPartition ssp = e.getKey(); Long offset = e.getValue(); TopicAndPartition tp = new TopicAndPartition(ssp.getStream(), ssp.getPartition().getPartitionId()); Long lag = latestLags.get(ssp); LOG.trace("Latest offset of {} is {}; lag = {}", ssp, offset, lag); if (lag != null && offset != null && lag >= 0) { long streamEndOffset = offset.longValue() + lag.longValue(); // update the metrics kafkaConsumerMetrics.setHighWatermarkValue(tp, streamEndOffset); kafkaConsumerMetrics.setLagValue(tp, lag.longValue()); } } }
private void refreshLagMetrics() { for (Map.Entry<SystemStreamPartition, Long> e : nextOffsets.entrySet()) { SystemStreamPartition ssp = e.getKey(); Long offset = e.getValue(); TopicAndPartition tp = new TopicAndPartition(ssp.getStream(), ssp.getPartition().getPartitionId()); Long lag = latestLags.get(ssp); LOG.trace("Latest offset of {} is {}; lag = {}", ssp, offset, lag); if (lag != null && offset != null && lag >= 0) { long streamEndOffset = offset.longValue() + lag.longValue(); // update the metrics kafkaConsumerMetrics.setHighWatermarkValue(tp, streamEndOffset); kafkaConsumerMetrics.setLagValue(tp, lag.longValue()); } } }
private void updateMetrics(ConsumerRecord<K, V> r, TopicPartition tp) { TopicAndPartition tap = KafkaSystemConsumer.toTopicAndPartition(tp); SystemStreamPartition ssp = new SystemStreamPartition(systemName, tp.topic(), new Partition(tp.partition())); Long lag = latestLags.get(ssp); if (lag == null) { throw new SamzaException("Unknown/unregistered ssp in latestLags. ssp=" + ssp + "; system=" + systemName); } long currentSSPLag = lag.longValue(); // lag between the current offset and the highwatermark if (currentSSPLag < 0) { return; } long recordOffset = r.offset(); long highWatermark = recordOffset + currentSSPLag; // derived value for the highwatermark int size = getRecordSize(r); kafkaConsumerMetrics.incReads(tap); kafkaConsumerMetrics.incBytesReads(tap, size); kafkaConsumerMetrics.setOffsets(tap, recordOffset); kafkaConsumerMetrics.incClientBytesReads(metricName, size); kafkaConsumerMetrics.setHighWatermarkValue(tap, highWatermark); }
private void updateMetrics(ConsumerRecord<K, V> r, TopicPartition tp) { TopicAndPartition tap = KafkaSystemConsumer.toTopicAndPartition(tp); SystemStreamPartition ssp = new SystemStreamPartition(systemName, tp.topic(), new Partition(tp.partition())); Long lag = latestLags.get(ssp); if (lag == null) { throw new SamzaException("Unknown/unregistered ssp in latestLags. ssp=" + ssp + "; system=" + systemName); } long currentSSPLag = lag.longValue(); // lag between the current offset and the highwatermark if (currentSSPLag < 0) { return; } long recordOffset = r.offset(); long highWatermark = recordOffset + currentSSPLag; // derived value for the highwatermark int size = getRecordSize(r); kafkaConsumerMetrics.incReads(tap); kafkaConsumerMetrics.incBytesReads(tap, size); kafkaConsumerMetrics.setOffsets(tap, recordOffset); kafkaConsumerMetrics.incClientBytesReads(metricName, size); kafkaConsumerMetrics.setHighWatermarkValue(tap, highWatermark); }
private void updateMetrics(ConsumerRecord<K, V> r, TopicPartition tp) { TopicAndPartition tap = KafkaSystemConsumer.toTopicAndPartition(tp); SystemStreamPartition ssp = new SystemStreamPartition(systemName, tp.topic(), new Partition(tp.partition())); Long lag = latestLags.get(ssp); if (lag == null) { throw new SamzaException("Unknown/unregistered ssp in latestLags. ssp=" + ssp + "; system=" + systemName); } long currentSSPLag = lag.longValue(); // lag between the current offset and the highwatermark if (currentSSPLag < 0) { return; } long recordOffset = r.offset(); long highWatermark = recordOffset + currentSSPLag; // derived value for the highwatermark int size = getRecordSize(r); kafkaConsumerMetrics.incReads(tap); kafkaConsumerMetrics.incBytesReads(tap, size); kafkaConsumerMetrics.setOffsets(tap, recordOffset); kafkaConsumerMetrics.incClientBytesReads(metricName, size); kafkaConsumerMetrics.setHighWatermarkValue(tap, highWatermark); }
metrics.setHighWatermarkValue(tp2, 1000); metrics.setHighWatermarkValue(tp2, 1001); // final value 1001 expectedValues.put(metrics.highWatermark().get(tp2).getName(), "1001");