KafkaConsumerProxy(Consumer<K, V> kafkaConsumer, String systemName, String clientId, KafkaSystemConsumer.KafkaConsumerMessageSink messageSink, KafkaSystemConsumerMetrics samzaConsumerMetrics, String metricName) { this.kafkaConsumer = kafkaConsumer; this.systemName = systemName; this.sink = messageSink; this.kafkaConsumerMetrics = samzaConsumerMetrics; this.metricName = metricName; this.clientId = clientId; this.kafkaConsumerMetrics.registerClientProxy(metricName); consumerPollThread = new Thread(createProxyThreadRunnable()); consumerPollThread.setDaemon(true); consumerPollThread.setName( "Samza KafkaConsumerProxy Poll " + consumerPollThread.getName() + " - " + systemName); LOG.info("Creating KafkaConsumerProxy with systeName={}, clientId={}, metricsName={}", systemName, clientId, metricName); }
private void refreshLagMetrics() { for (Map.Entry<SystemStreamPartition, Long> e : nextOffsets.entrySet()) { SystemStreamPartition ssp = e.getKey(); Long offset = e.getValue(); TopicAndPartition tp = new TopicAndPartition(ssp.getStream(), ssp.getPartition().getPartitionId()); Long lag = latestLags.get(ssp); LOG.trace("Latest offset of {} is {}; lag = {}", ssp, offset, lag); if (lag != null && offset != null && lag >= 0) { long streamEndOffset = offset.longValue() + lag.longValue(); // update the metrics kafkaConsumerMetrics.setHighWatermarkValue(tp, streamEndOffset); kafkaConsumerMetrics.setLagValue(tp, lag.longValue()); } } }
/** * Add new partition to the list of polled partitions. * Bust only be called before {@link KafkaConsumerProxy#start} is called.. */ public void addTopicPartition(SystemStreamPartition ssp, long nextOffset) { LOG.info(String.format("Adding new topicPartition %s with offset %s to queue for consumer %s", ssp, nextOffset, this)); topicPartitionToSSP.put(KafkaSystemConsumer.toTopicPartition(ssp), ssp); //registered SSPs // this is already vetted offset so there is no need to validate it nextOffsets.put(ssp, nextOffset); kafkaConsumerMetrics.setNumTopicPartitions(metricName, nextOffsets.size()); }
KafkaSystemConsumerMetrics metrics = new KafkaSystemConsumerMetrics(systemName, registry); metrics.registerTopicAndPartition(tp1); metrics.registerTopicAndPartition(tp2); metrics.registerClientProxy(clientName); metrics.setOffsets(tp1, 1001); metrics.setOffsets(tp2, 1002); expectedValues.put(metrics.offsets().get(tp1).getName(), "1001"); expectedValues.put(metrics.offsets().get(tp2).getName(), "1002"); metrics.incBytesReads(tp1, 10); metrics.incBytesReads(tp1, 5); // total 15 expectedValues.put(metrics.bytesRead().get(tp1).getName(), "15"); metrics.incReads(tp1); metrics.incReads(tp1); // total 2 expectedValues.put(metrics.reads().get(tp1).getName(), "2"); metrics.setHighWatermarkValue(tp2, 1000); metrics.setHighWatermarkValue(tp2, 1001); // final value 1001 expectedValues.put(metrics.highWatermark().get(tp2).getName(), "1001"); metrics.setLagValue(tp1, 200); metrics.setLagValue(tp1, 201); // final value 201 expectedValues.put(metrics.lag().get(tp1).getName(), "201"); metrics.incClientBytesReads(clientName, 100); // broker-bytes-read
private void updateMetrics(ConsumerRecord<K, V> r, TopicPartition tp) { TopicAndPartition tap = KafkaSystemConsumer.toTopicAndPartition(tp); SystemStreamPartition ssp = new SystemStreamPartition(systemName, tp.topic(), new Partition(tp.partition())); Long lag = latestLags.get(ssp); if (lag == null) { throw new SamzaException("Unknown/unregistered ssp in latestLags. ssp=" + ssp + "; system=" + systemName); } long currentSSPLag = lag.longValue(); // lag between the current offset and the highwatermark if (currentSSPLag < 0) { return; } long recordOffset = r.offset(); long highWatermark = recordOffset + currentSSPLag; // derived value for the highwatermark int size = getRecordSize(r); kafkaConsumerMetrics.incReads(tap); kafkaConsumerMetrics.incBytesReads(tap, size); kafkaConsumerMetrics.setOffsets(tap, recordOffset); kafkaConsumerMetrics.incClientBytesReads(metricName, size); kafkaConsumerMetrics.setHighWatermarkValue(tap, highWatermark); }
kafkaConsumerMetrics.incClientReads(metricName); SLEEP_MS_WHILE_NO_TOPIC_PARTITION); kafkaConsumerMetrics.incClientSkippedFetchRequests(metricName);
/** * record the ssp and the offset. Do not submit it to the consumer yet. * @param systemStreamPartition ssp to register * @param offset offset to register with */ @Override public void register(SystemStreamPartition systemStreamPartition, String offset) { if (started.get()) { String msg = String.format("%s: Trying to register partition after consumer has been started. ssp=%s", this, systemStreamPartition); throw new SamzaException(msg); } if (!systemStreamPartition.getSystem().equals(systemName)) { LOG.warn("{}: ignoring SSP {}, because this consumer's system doesn't match.", this, systemStreamPartition); return; } LOG.info("{}: Registering ssp = {} with offset {}", this, systemStreamPartition, offset); super.register(systemStreamPartition, offset); TopicPartition tp = toTopicPartition(systemStreamPartition); topicPartitionsToSSP.put(tp, systemStreamPartition); String existingOffset = topicPartitionsToOffset.get(tp); // register the older (of the two) offset in the consumer, to guarantee we do not miss any messages. if (existingOffset == null || compareOffsets(existingOffset, offset) > 0) { topicPartitionsToOffset.put(tp, offset); } metrics.registerTopicAndPartition(toTopicAndPartition(tp)); }
private void updateMetrics(ConsumerRecord<K, V> r, TopicPartition tp) { TopicAndPartition tap = KafkaSystemConsumer.toTopicAndPartition(tp); SystemStreamPartition ssp = new SystemStreamPartition(systemName, tp.topic(), new Partition(tp.partition())); Long lag = latestLags.get(ssp); if (lag == null) { throw new SamzaException("Unknown/unregistered ssp in latestLags. ssp=" + ssp + "; system=" + systemName); } long currentSSPLag = lag.longValue(); // lag between the current offset and the highwatermark if (currentSSPLag < 0) { return; } long recordOffset = r.offset(); long highWatermark = recordOffset + currentSSPLag; // derived value for the highwatermark int size = getRecordSize(r); kafkaConsumerMetrics.incReads(tap); kafkaConsumerMetrics.incBytesReads(tap, size); kafkaConsumerMetrics.setOffsets(tap, recordOffset); kafkaConsumerMetrics.incClientBytesReads(metricName, size); kafkaConsumerMetrics.setHighWatermarkValue(tap, highWatermark); }
kafkaConsumerMetrics.incClientReads(metricName); SLEEP_MS_WHILE_NO_TOPIC_PARTITION); kafkaConsumerMetrics.incClientSkippedFetchRequests(metricName);
/** * record the ssp and the offset. Do not submit it to the consumer yet. * @param systemStreamPartition ssp to register * @param offset offset to register with */ @Override public void register(SystemStreamPartition systemStreamPartition, String offset) { if (started.get()) { String msg = String.format("%s: Trying to register partition after consumer has been started. ssp=%s", this, systemStreamPartition); throw new SamzaException(msg); } if (!systemStreamPartition.getSystem().equals(systemName)) { LOG.warn("{}: ignoring SSP {}, because this consumer's system doesn't match.", this, systemStreamPartition); return; } LOG.info("{}: Registering ssp = {} with offset {}", this, systemStreamPartition, offset); super.register(systemStreamPartition, offset); TopicPartition tp = toTopicPartition(systemStreamPartition); topicPartitionsToSSP.put(tp, systemStreamPartition); String existingOffset = topicPartitionsToOffset.get(tp); // register the older (of the two) offset in the consumer, to guarantee we do not miss any messages. if (existingOffset == null || compareOffsets(existingOffset, offset) > 0) { topicPartitionsToOffset.put(tp, offset); } metrics.registerTopicAndPartition(toTopicAndPartition(tp)); }
private void updateMetrics(ConsumerRecord<K, V> r, TopicPartition tp) { TopicAndPartition tap = KafkaSystemConsumer.toTopicAndPartition(tp); SystemStreamPartition ssp = new SystemStreamPartition(systemName, tp.topic(), new Partition(tp.partition())); Long lag = latestLags.get(ssp); if (lag == null) { throw new SamzaException("Unknown/unregistered ssp in latestLags. ssp=" + ssp + "; system=" + systemName); } long currentSSPLag = lag.longValue(); // lag between the current offset and the highwatermark if (currentSSPLag < 0) { return; } long recordOffset = r.offset(); long highWatermark = recordOffset + currentSSPLag; // derived value for the highwatermark int size = getRecordSize(r); kafkaConsumerMetrics.incReads(tap); kafkaConsumerMetrics.incBytesReads(tap, size); kafkaConsumerMetrics.setOffsets(tap, recordOffset); kafkaConsumerMetrics.incClientBytesReads(metricName, size); kafkaConsumerMetrics.setHighWatermarkValue(tap, highWatermark); }
private void refreshLagMetrics() { for (Map.Entry<SystemStreamPartition, Long> e : nextOffsets.entrySet()) { SystemStreamPartition ssp = e.getKey(); Long offset = e.getValue(); TopicAndPartition tp = new TopicAndPartition(ssp.getStream(), ssp.getPartition().getPartitionId()); Long lag = latestLags.get(ssp); LOG.trace("Latest offset of {} is {}; lag = {}", ssp, offset, lag); if (lag != null && offset != null && lag >= 0) { long streamEndOffset = offset.longValue() + lag.longValue(); // update the metrics kafkaConsumerMetrics.setHighWatermarkValue(tp, streamEndOffset); kafkaConsumerMetrics.setLagValue(tp, lag.longValue()); } } }
kafkaConsumerMetrics.incClientReads(metricName); SLEEP_MS_WHILE_NO_TOPIC_PARTITION); kafkaConsumerMetrics.incClientSkippedFetchRequests(metricName);
/** * Add new partition to the list of polled partitions. * Bust only be called before {@link KafkaConsumerProxy#start} is called.. */ public void addTopicPartition(SystemStreamPartition ssp, long nextOffset) { LOG.info(String.format("Adding new topicPartition %s with offset %s to queue for consumer %s", ssp, nextOffset, this)); topicPartitionToSSP.put(KafkaSystemConsumer.toTopicPartition(ssp), ssp); //registered SSPs // this is already vetted offset so there is no need to validate it nextOffsets.put(ssp, nextOffset); kafkaConsumerMetrics.setNumTopicPartitions(metricName, nextOffsets.size()); }
KafkaConsumerProxy(Consumer<K, V> kafkaConsumer, String systemName, String clientId, KafkaSystemConsumer.KafkaConsumerMessageSink messageSink, KafkaSystemConsumerMetrics samzaConsumerMetrics, String metricName) { this.kafkaConsumer = kafkaConsumer; this.systemName = systemName; this.sink = messageSink; this.kafkaConsumerMetrics = samzaConsumerMetrics; this.metricName = metricName; this.clientId = clientId; this.kafkaConsumerMetrics.registerClientProxy(metricName); consumerPollThread = new Thread(createProxyThreadRunnable()); consumerPollThread.setDaemon(true); consumerPollThread.setName( "Samza KafkaConsumerProxy Poll " + consumerPollThread.getName() + " - " + systemName); LOG.info("Creating KafkaConsumerProxy with systeName={}, clientId={}, metricsName={}", systemName, clientId, metricName); }
/** * record the ssp and the offset. Do not submit it to the consumer yet. * @param systemStreamPartition ssp to register * @param offset offset to register with */ @Override public void register(SystemStreamPartition systemStreamPartition, String offset) { if (started.get()) { String msg = String.format("%s: Trying to register partition after consumer has been started. ssp=%s", this, systemStreamPartition); throw new SamzaException(msg); } if (!systemStreamPartition.getSystem().equals(systemName)) { LOG.warn("{}: ignoring SSP {}, because this consumer's system doesn't match.", this, systemStreamPartition); return; } LOG.info("{}: Registering ssp = {} with offset {}", this, systemStreamPartition, offset); super.register(systemStreamPartition, offset); TopicPartition tp = toTopicPartition(systemStreamPartition); topicPartitionsToSSP.put(tp, systemStreamPartition); String existingOffset = topicPartitionsToOffset.get(tp); // register the older (of the two) offset in the consumer, to guarantee we do not miss any messages. if (existingOffset == null || compareOffsets(existingOffset, offset) > 0) { topicPartitionsToOffset.put(tp, offset); } metrics.registerTopicAndPartition(toTopicAndPartition(tp)); }
private void refreshLagMetrics() { for (Map.Entry<SystemStreamPartition, Long> e : nextOffsets.entrySet()) { SystemStreamPartition ssp = e.getKey(); Long offset = e.getValue(); TopicAndPartition tp = new TopicAndPartition(ssp.getStream(), ssp.getPartition().getPartitionId()); Long lag = latestLags.get(ssp); LOG.trace("Latest offset of {} is {}; lag = {}", ssp, offset, lag); if (lag != null && offset != null && lag >= 0) { long streamEndOffset = offset.longValue() + lag.longValue(); // update the metrics kafkaConsumerMetrics.setHighWatermarkValue(tp, streamEndOffset); kafkaConsumerMetrics.setLagValue(tp, lag.longValue()); } } }
/** * Add new partition to the list of polled partitions. * Bust only be called before {@link KafkaConsumerProxy#start} is called.. */ public void addTopicPartition(SystemStreamPartition ssp, long nextOffset) { LOG.info(String.format("Adding new topicPartition %s with offset %s to queue for consumer %s", ssp, nextOffset, this)); topicPartitionToSSP.put(KafkaSystemConsumer.toTopicPartition(ssp), ssp); //registered SSPs // this is already vetted offset so there is no need to validate it nextOffsets.put(ssp, nextOffset); kafkaConsumerMetrics.setNumTopicPartitions(metricName, nextOffsets.size()); }
KafkaConsumerProxy(Consumer<K, V> kafkaConsumer, String systemName, String clientId, KafkaSystemConsumer.KafkaConsumerMessageSink messageSink, KafkaSystemConsumerMetrics samzaConsumerMetrics, String metricName) { this.kafkaConsumer = kafkaConsumer; this.systemName = systemName; this.sink = messageSink; this.kafkaConsumerMetrics = samzaConsumerMetrics; this.metricName = metricName; this.clientId = clientId; this.kafkaConsumerMetrics.registerClientProxy(metricName); consumerPollThread = new Thread(createProxyThreadRunnable()); consumerPollThread.setDaemon(true); consumerPollThread.setName( "Samza KafkaConsumerProxy Poll " + consumerPollThread.getName() + " - " + systemName); LOG.info("Creating KafkaConsumerProxy with systeName={}, clientId={}, metricsName={}", systemName, clientId, metricName); }