Tabnine Logo
KafkaSystemConsumer$KafkaConsumerMessageSink.setIsAtHighWatermark
Code IndexAdd Tabnine to your IDE (free)

How to use
setIsAtHighWatermark
method
in
org.apache.samza.system.kafka.KafkaSystemConsumer$KafkaConsumerMessageSink

Best Java code snippets using org.apache.samza.system.kafka.KafkaSystemConsumer$KafkaConsumerMessageSink.setIsAtHighWatermark (Showing top 6 results out of 315)

origin: apache/samza

private void initializeLags() {
 // This is expensive, so only do it once at the beginning. After the first poll, we can rely on metrics for lag.
 Map<TopicPartition, Long> endOffsets = kafkaConsumer.endOffsets(topicPartitionToSSP.keySet());
 endOffsets.forEach((tp, offset) -> {
  SystemStreamPartition ssp = topicPartitionToSSP.get(tp);
  long startingOffset = nextOffsets.get(ssp);
  // End offsets are the offset of the newest message + 1
  // If the message we are about to consume is < end offset, we are starting with a lag.
  long initialLag = endOffsets.get(tp) - startingOffset;
  LOG.info("Initial lag for SSP {} is {} (end={}, startOffset={})", ssp, initialLag, endOffsets.get(tp), startingOffset);
  latestLags.put(ssp, initialLag);
  sink.setIsAtHighWatermark(ssp, initialLag == 0);
 });
 // initialize lag metrics
 refreshLagMetrics();
}
origin: org.apache.samza/samza-kafka_2.11

private void initializeLags() {
 // This is expensive, so only do it once at the beginning. After the first poll, we can rely on metrics for lag.
 Map<TopicPartition, Long> endOffsets = kafkaConsumer.endOffsets(topicPartitionToSSP.keySet());
 endOffsets.forEach((tp, offset) -> {
  SystemStreamPartition ssp = topicPartitionToSSP.get(tp);
  long startingOffset = nextOffsets.get(ssp);
  // End offsets are the offset of the newest message + 1
  // If the message we are about to consume is < end offset, we are starting with a lag.
  long initialLag = endOffsets.get(tp) - startingOffset;
  LOG.info("Initial lag for SSP {} is {} (end={}, startOffset={})", ssp, initialLag, endOffsets.get(tp), startingOffset);
  latestLags.put(ssp, initialLag);
  sink.setIsAtHighWatermark(ssp, initialLag == 0);
 });
 // initialize lag metrics
 refreshLagMetrics();
}
origin: org.apache.samza/samza-kafka

private void initializeLags() {
 // This is expensive, so only do it once at the beginning. After the first poll, we can rely on metrics for lag.
 Map<TopicPartition, Long> endOffsets = kafkaConsumer.endOffsets(topicPartitionToSSP.keySet());
 endOffsets.forEach((tp, offset) -> {
  SystemStreamPartition ssp = topicPartitionToSSP.get(tp);
  long startingOffset = nextOffsets.get(ssp);
  // End offsets are the offset of the newest message + 1
  // If the message we are about to consume is < end offset, we are starting with a lag.
  long initialLag = endOffsets.get(tp) - startingOffset;
  LOG.info("Initial lag for SSP {} is {} (end={}, startOffset={})", ssp, initialLag, endOffsets.get(tp), startingOffset);
  latestLags.put(ssp, initialLag);
  sink.setIsAtHighWatermark(ssp, initialLag == 0);
 });
 // initialize lag metrics
 refreshLagMetrics();
}
origin: org.apache.samza/samza-kafka_2.11

private void populateCurrentLags(Set<SystemStreamPartition> ssps) {
 Map<MetricName, ? extends Metric> consumerMetrics = kafkaConsumer.metrics();
 // populate the MetricNames first time
 if (perPartitionMetrics.isEmpty()) {
  HashMap<String, String> tags = new HashMap<>();
  tags.put("client-id", clientId); // this is required by the KafkaConsumer to get the metrics
  for (SystemStreamPartition ssp : ssps) {
   TopicPartition tp = KafkaSystemConsumer.toTopicPartition(ssp);
   perPartitionMetrics.put(ssp, new MetricName(tp + ".records-lag", "consumer-fetch-manager-metrics", "", tags));
  }
 }
 for (SystemStreamPartition ssp : ssps) {
  MetricName mn = perPartitionMetrics.get(ssp);
  Metric currentLagMetric = consumerMetrics.get(mn);
  // High watermark is fixed to be the offset of last available message,
  // so the lag is now at least 0, which is the same as Samza's definition.
  // If the lag is not 0, then isAtHead is not true, and kafkaClient keeps polling.
  long currentLag = (currentLagMetric != null) ? (long) currentLagMetric.value() : -1L;
  latestLags.put(ssp, currentLag);
  // calls the setIsAtHead for the BlockingEnvelopeMap
  sink.setIsAtHighWatermark(ssp, currentLag == 0);
 }
}
origin: org.apache.samza/samza-kafka

private void populateCurrentLags(Set<SystemStreamPartition> ssps) {
 Map<MetricName, ? extends Metric> consumerMetrics = kafkaConsumer.metrics();
 // populate the MetricNames first time
 if (perPartitionMetrics.isEmpty()) {
  HashMap<String, String> tags = new HashMap<>();
  tags.put("client-id", clientId); // this is required by the KafkaConsumer to get the metrics
  for (SystemStreamPartition ssp : ssps) {
   TopicPartition tp = KafkaSystemConsumer.toTopicPartition(ssp);
   perPartitionMetrics.put(ssp, new MetricName(tp + ".records-lag", "consumer-fetch-manager-metrics", "", tags));
  }
 }
 for (SystemStreamPartition ssp : ssps) {
  MetricName mn = perPartitionMetrics.get(ssp);
  Metric currentLagMetric = consumerMetrics.get(mn);
  // High watermark is fixed to be the offset of last available message,
  // so the lag is now at least 0, which is the same as Samza's definition.
  // If the lag is not 0, then isAtHead is not true, and kafkaClient keeps polling.
  long currentLag = (currentLagMetric != null) ? (long) currentLagMetric.value() : -1L;
  latestLags.put(ssp, currentLag);
  // calls the setIsAtHead for the BlockingEnvelopeMap
  sink.setIsAtHighWatermark(ssp, currentLag == 0);
 }
}
origin: apache/samza

private void populateCurrentLags(Set<SystemStreamPartition> ssps) {
 Map<MetricName, ? extends Metric> consumerMetrics = kafkaConsumer.metrics();
 // populate the MetricNames first time
 if (perPartitionMetrics.isEmpty()) {
  HashMap<String, String> tags = new HashMap<>();
  tags.put("client-id", clientId); // this is required by the KafkaConsumer to get the metrics
  for (SystemStreamPartition ssp : ssps) {
   TopicPartition tp = KafkaSystemConsumer.toTopicPartition(ssp);
   perPartitionMetrics.put(ssp, new MetricName(tp + ".records-lag", "consumer-fetch-manager-metrics", "", tags));
  }
 }
 for (SystemStreamPartition ssp : ssps) {
  MetricName mn = perPartitionMetrics.get(ssp);
  Metric currentLagMetric = consumerMetrics.get(mn);
  // High watermark is fixed to be the offset of last available message,
  // so the lag is now at least 0, which is the same as Samza's definition.
  // If the lag is not 0, then isAtHead is not true, and kafkaClient keeps polling.
  long currentLag = (currentLagMetric != null) ? (long) currentLagMetric.value() : -1L;
  latestLags.put(ssp, currentLag);
  // calls the setIsAtHead for the BlockingEnvelopeMap
  sink.setIsAtHighWatermark(ssp, currentLag == 0);
 }
}
org.apache.samza.system.kafkaKafkaSystemConsumer$KafkaConsumerMessageSinksetIsAtHighWatermark

Popular methods of KafkaSystemConsumer$KafkaConsumerMessageSink

  • addMessage
  • needsMoreMessages
  • <init>

Popular in Java

  • Reactive rest calls using spring rest template
  • scheduleAtFixedRate (ScheduledExecutorService)
  • runOnUiThread (Activity)
  • notifyDataSetChanged (ArrayAdapter)
  • File (java.io)
    An "abstract" representation of a file system entity identified by a pathname. The pathname may be a
  • BigInteger (java.math)
    An immutable arbitrary-precision signed integer.FAST CRYPTOGRAPHY This implementation is efficient f
  • SimpleDateFormat (java.text)
    Formats and parses dates in a locale-sensitive manner. Formatting turns a Date into a String, and pa
  • Hashtable (java.util)
    A plug-in replacement for JDK1.5 java.util.Hashtable. This version is based on org.cliffc.high_scale
  • HttpServletRequest (javax.servlet.http)
    Extends the javax.servlet.ServletRequest interface to provide request information for HTTP servlets.
  • IOUtils (org.apache.commons.io)
    General IO stream manipulation utilities. This class provides static utility methods for input/outpu
  • Top plugins for WebStorm
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now