Tabnine Logo
KafkaSystemConsumerMetrics.setHighWatermarkValue
Code IndexAdd Tabnine to your IDE (free)

How to use
setHighWatermarkValue
method
in
org.apache.samza.system.kafka.KafkaSystemConsumerMetrics

Best Java code snippets using org.apache.samza.system.kafka.KafkaSystemConsumerMetrics.setHighWatermarkValue (Showing top 7 results out of 315)

origin: org.apache.samza/samza-kafka

private void refreshLagMetrics() {
 for (Map.Entry<SystemStreamPartition, Long> e : nextOffsets.entrySet()) {
  SystemStreamPartition ssp = e.getKey();
  Long offset = e.getValue();
  TopicAndPartition tp = new TopicAndPartition(ssp.getStream(), ssp.getPartition().getPartitionId());
  Long lag = latestLags.get(ssp);
  LOG.trace("Latest offset of {} is  {}; lag = {}", ssp, offset, lag);
  if (lag != null && offset != null && lag >= 0) {
   long streamEndOffset = offset.longValue() + lag.longValue();
   // update the metrics
   kafkaConsumerMetrics.setHighWatermarkValue(tp, streamEndOffset);
   kafkaConsumerMetrics.setLagValue(tp, lag.longValue());
  }
 }
}
origin: apache/samza

private void refreshLagMetrics() {
 for (Map.Entry<SystemStreamPartition, Long> e : nextOffsets.entrySet()) {
  SystemStreamPartition ssp = e.getKey();
  Long offset = e.getValue();
  TopicAndPartition tp = new TopicAndPartition(ssp.getStream(), ssp.getPartition().getPartitionId());
  Long lag = latestLags.get(ssp);
  LOG.trace("Latest offset of {} is  {}; lag = {}", ssp, offset, lag);
  if (lag != null && offset != null && lag >= 0) {
   long streamEndOffset = offset.longValue() + lag.longValue();
   // update the metrics
   kafkaConsumerMetrics.setHighWatermarkValue(tp, streamEndOffset);
   kafkaConsumerMetrics.setLagValue(tp, lag.longValue());
  }
 }
}
origin: org.apache.samza/samza-kafka_2.11

private void refreshLagMetrics() {
 for (Map.Entry<SystemStreamPartition, Long> e : nextOffsets.entrySet()) {
  SystemStreamPartition ssp = e.getKey();
  Long offset = e.getValue();
  TopicAndPartition tp = new TopicAndPartition(ssp.getStream(), ssp.getPartition().getPartitionId());
  Long lag = latestLags.get(ssp);
  LOG.trace("Latest offset of {} is  {}; lag = {}", ssp, offset, lag);
  if (lag != null && offset != null && lag >= 0) {
   long streamEndOffset = offset.longValue() + lag.longValue();
   // update the metrics
   kafkaConsumerMetrics.setHighWatermarkValue(tp, streamEndOffset);
   kafkaConsumerMetrics.setLagValue(tp, lag.longValue());
  }
 }
}
origin: org.apache.samza/samza-kafka

private void updateMetrics(ConsumerRecord<K, V> r, TopicPartition tp) {
 TopicAndPartition tap = KafkaSystemConsumer.toTopicAndPartition(tp);
 SystemStreamPartition ssp = new SystemStreamPartition(systemName, tp.topic(), new Partition(tp.partition()));
 Long lag = latestLags.get(ssp);
 if (lag == null) {
  throw new SamzaException("Unknown/unregistered ssp in latestLags. ssp=" + ssp + "; system=" + systemName);
 }
 long currentSSPLag = lag.longValue(); // lag between the current offset and the highwatermark
 if (currentSSPLag < 0) {
  return;
 }
 long recordOffset = r.offset();
 long highWatermark = recordOffset + currentSSPLag; // derived value for the highwatermark
 int size = getRecordSize(r);
 kafkaConsumerMetrics.incReads(tap);
 kafkaConsumerMetrics.incBytesReads(tap, size);
 kafkaConsumerMetrics.setOffsets(tap, recordOffset);
 kafkaConsumerMetrics.incClientBytesReads(metricName, size);
 kafkaConsumerMetrics.setHighWatermarkValue(tap, highWatermark);
}
origin: apache/samza

private void updateMetrics(ConsumerRecord<K, V> r, TopicPartition tp) {
 TopicAndPartition tap = KafkaSystemConsumer.toTopicAndPartition(tp);
 SystemStreamPartition ssp = new SystemStreamPartition(systemName, tp.topic(), new Partition(tp.partition()));
 Long lag = latestLags.get(ssp);
 if (lag == null) {
  throw new SamzaException("Unknown/unregistered ssp in latestLags. ssp=" + ssp + "; system=" + systemName);
 }
 long currentSSPLag = lag.longValue(); // lag between the current offset and the highwatermark
 if (currentSSPLag < 0) {
  return;
 }
 long recordOffset = r.offset();
 long highWatermark = recordOffset + currentSSPLag; // derived value for the highwatermark
 int size = getRecordSize(r);
 kafkaConsumerMetrics.incReads(tap);
 kafkaConsumerMetrics.incBytesReads(tap, size);
 kafkaConsumerMetrics.setOffsets(tap, recordOffset);
 kafkaConsumerMetrics.incClientBytesReads(metricName, size);
 kafkaConsumerMetrics.setHighWatermarkValue(tap, highWatermark);
}
origin: org.apache.samza/samza-kafka_2.11

private void updateMetrics(ConsumerRecord<K, V> r, TopicPartition tp) {
 TopicAndPartition tap = KafkaSystemConsumer.toTopicAndPartition(tp);
 SystemStreamPartition ssp = new SystemStreamPartition(systemName, tp.topic(), new Partition(tp.partition()));
 Long lag = latestLags.get(ssp);
 if (lag == null) {
  throw new SamzaException("Unknown/unregistered ssp in latestLags. ssp=" + ssp + "; system=" + systemName);
 }
 long currentSSPLag = lag.longValue(); // lag between the current offset and the highwatermark
 if (currentSSPLag < 0) {
  return;
 }
 long recordOffset = r.offset();
 long highWatermark = recordOffset + currentSSPLag; // derived value for the highwatermark
 int size = getRecordSize(r);
 kafkaConsumerMetrics.incReads(tap);
 kafkaConsumerMetrics.incBytesReads(tap, size);
 kafkaConsumerMetrics.setOffsets(tap, recordOffset);
 kafkaConsumerMetrics.incClientBytesReads(metricName, size);
 kafkaConsumerMetrics.setHighWatermarkValue(tap, highWatermark);
}
origin: apache/samza

metrics.setHighWatermarkValue(tp2, 1000);
metrics.setHighWatermarkValue(tp2, 1001); // final value 1001
expectedValues.put(metrics.highWatermark().get(tp2).getName(), "1001");
org.apache.samza.system.kafkaKafkaSystemConsumerMetricssetHighWatermarkValue

Popular methods of KafkaSystemConsumerMetrics

  • incBytesReads
  • incClientBytesReads
  • incClientReads
  • incReads
  • registerClientProxy
  • registerTopicAndPartition
  • setLagValue
  • setNumTopicPartitions
  • setOffsets
  • incClientSkippedFetchRequests
  • registry
  • <init>
  • registry,
  • <init>,
  • bytesRead,
  • clientBytesRead,
  • clientReads,
  • group,
  • highWatermark,
  • lag,
  • offsets

Popular in Java

  • Updating database using SQL prepared statement
  • startActivity (Activity)
  • setRequestProperty (URLConnection)
  • notifyDataSetChanged (ArrayAdapter)
  • BufferedReader (java.io)
    Wraps an existing Reader and buffers the input. Expensive interaction with the underlying reader is
  • RandomAccessFile (java.io)
    Allows reading from and writing to a file in a random-access manner. This is different from the uni-
  • SocketTimeoutException (java.net)
    This exception is thrown when a timeout expired on a socket read or accept operation.
  • URLConnection (java.net)
    A connection to a URL for reading or writing. For HTTP connections, see HttpURLConnection for docume
  • Handler (java.util.logging)
    A Handler object accepts a logging request and exports the desired messages to a target, for example
  • Logger (org.apache.log4j)
    This is the central class in the log4j package. Most logging operations, except configuration, are d
  • Top 17 Plugins for Android Studio
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimAtomGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyStudentsTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now