How to use

toTopicAndPartition

method

in

org.apache.samza.system.kafka.KafkaSystemConsumer

/**
 * record the ssp and the offset. Do not submit it to the consumer yet.
 * @param systemStreamPartition ssp to register
 * @param offset offset to register with
 */
@Override
public void register(SystemStreamPartition systemStreamPartition, String offset) {
 if (started.get()) {
  String msg = String.format("%s: Trying to register partition after consumer has been started. ssp=%s", this,
    systemStreamPartition);
  throw new SamzaException(msg);
 }
 if (!systemStreamPartition.getSystem().equals(systemName)) {
  LOG.warn("{}: ignoring SSP {}, because this consumer's system doesn't match.", this, systemStreamPartition);
  return;
 }
 LOG.info("{}: Registering ssp = {} with offset {}", this, systemStreamPartition, offset);
 super.register(systemStreamPartition, offset);
 TopicPartition tp = toTopicPartition(systemStreamPartition);
 topicPartitionsToSSP.put(tp, systemStreamPartition);
 String existingOffset = topicPartitionsToOffset.get(tp);
 // register the older (of the two) offset in the consumer, to guarantee we do not miss any messages.
 if (existingOffset == null || compareOffsets(existingOffset, offset) > 0) {
  topicPartitionsToOffset.put(tp, offset);
 }
 metrics.registerTopicAndPartition(toTopicAndPartition(tp));
}

/**
 * record the ssp and the offset. Do not submit it to the consumer yet.
 * @param systemStreamPartition ssp to register
 * @param offset offset to register with
 */
@Override
public void register(SystemStreamPartition systemStreamPartition, String offset) {
 if (started.get()) {
  String msg = String.format("%s: Trying to register partition after consumer has been started. ssp=%s", this,
    systemStreamPartition);
  throw new SamzaException(msg);
 }
 if (!systemStreamPartition.getSystem().equals(systemName)) {
  LOG.warn("{}: ignoring SSP {}, because this consumer's system doesn't match.", this, systemStreamPartition);
  return;
 }
 LOG.info("{}: Registering ssp = {} with offset {}", this, systemStreamPartition, offset);
 super.register(systemStreamPartition, offset);
 TopicPartition tp = toTopicPartition(systemStreamPartition);
 topicPartitionsToSSP.put(tp, systemStreamPartition);
 String existingOffset = topicPartitionsToOffset.get(tp);
 // register the older (of the two) offset in the consumer, to guarantee we do not miss any messages.
 if (existingOffset == null || compareOffsets(existingOffset, offset) > 0) {
  topicPartitionsToOffset.put(tp, offset);
 }
 metrics.registerTopicAndPartition(toTopicAndPartition(tp));
}

/**
 * record the ssp and the offset. Do not submit it to the consumer yet.
 * @param systemStreamPartition ssp to register
 * @param offset offset to register with
 */
@Override
public void register(SystemStreamPartition systemStreamPartition, String offset) {
 if (started.get()) {
  String msg = String.format("%s: Trying to register partition after consumer has been started. ssp=%s", this,
    systemStreamPartition);
  throw new SamzaException(msg);
 }
 if (!systemStreamPartition.getSystem().equals(systemName)) {
  LOG.warn("{}: ignoring SSP {}, because this consumer's system doesn't match.", this, systemStreamPartition);
  return;
 }
 LOG.info("{}: Registering ssp = {} with offset {}", this, systemStreamPartition, offset);
 super.register(systemStreamPartition, offset);
 TopicPartition tp = toTopicPartition(systemStreamPartition);
 topicPartitionsToSSP.put(tp, systemStreamPartition);
 String existingOffset = topicPartitionsToOffset.get(tp);
 // register the older (of the two) offset in the consumer, to guarantee we do not miss any messages.
 if (existingOffset == null || compareOffsets(existingOffset, offset) > 0) {
  topicPartitionsToOffset.put(tp, offset);
 }
 metrics.registerTopicAndPartition(toTopicAndPartition(tp));
}

private void updateMetrics(ConsumerRecord<K, V> r, TopicPartition tp) {
 TopicAndPartition tap = KafkaSystemConsumer.toTopicAndPartition(tp);
 SystemStreamPartition ssp = new SystemStreamPartition(systemName, tp.topic(), new Partition(tp.partition()));
 Long lag = latestLags.get(ssp);
 if (lag == null) {
  throw new SamzaException("Unknown/unregistered ssp in latestLags. ssp=" + ssp + "; system=" + systemName);
 }
 long currentSSPLag = lag.longValue(); // lag between the current offset and the highwatermark
 if (currentSSPLag < 0) {
  return;
 }
 long recordOffset = r.offset();
 long highWatermark = recordOffset + currentSSPLag; // derived value for the highwatermark
 int size = getRecordSize(r);
 kafkaConsumerMetrics.incReads(tap);
 kafkaConsumerMetrics.incBytesReads(tap, size);
 kafkaConsumerMetrics.setOffsets(tap, recordOffset);
 kafkaConsumerMetrics.incClientBytesReads(metricName, size);
 kafkaConsumerMetrics.setHighWatermarkValue(tap, highWatermark);
}

private void updateMetrics(ConsumerRecord<K, V> r, TopicPartition tp) {
 TopicAndPartition tap = KafkaSystemConsumer.toTopicAndPartition(tp);
 SystemStreamPartition ssp = new SystemStreamPartition(systemName, tp.topic(), new Partition(tp.partition()));
 Long lag = latestLags.get(ssp);
 if (lag == null) {
  throw new SamzaException("Unknown/unregistered ssp in latestLags. ssp=" + ssp + "; system=" + systemName);
 }
 long currentSSPLag = lag.longValue(); // lag between the current offset and the highwatermark
 if (currentSSPLag < 0) {
  return;
 }
 long recordOffset = r.offset();
 long highWatermark = recordOffset + currentSSPLag; // derived value for the highwatermark
 int size = getRecordSize(r);
 kafkaConsumerMetrics.incReads(tap);
 kafkaConsumerMetrics.incBytesReads(tap, size);
 kafkaConsumerMetrics.setOffsets(tap, recordOffset);
 kafkaConsumerMetrics.incClientBytesReads(metricName, size);
 kafkaConsumerMetrics.setHighWatermarkValue(tap, highWatermark);
}

private void updateMetrics(ConsumerRecord<K, V> r, TopicPartition tp) {
 TopicAndPartition tap = KafkaSystemConsumer.toTopicAndPartition(tp);
 SystemStreamPartition ssp = new SystemStreamPartition(systemName, tp.topic(), new Partition(tp.partition()));
 Long lag = latestLags.get(ssp);
 if (lag == null) {
  throw new SamzaException("Unknown/unregistered ssp in latestLags. ssp=" + ssp + "; system=" + systemName);
 }
 long currentSSPLag = lag.longValue(); // lag between the current offset and the highwatermark
 if (currentSSPLag < 0) {
  return;
 }
 long recordOffset = r.offset();
 long highWatermark = recordOffset + currentSSPLag; // derived value for the highwatermark
 int size = getRecordSize(r);
 kafkaConsumerMetrics.incReads(tap);
 kafkaConsumerMetrics.incBytesReads(tap, size);
 kafkaConsumerMetrics.setOffsets(tap, recordOffset);
 kafkaConsumerMetrics.incClientBytesReads(metricName, size);
 kafkaConsumerMetrics.setHighWatermarkValue(tap, highWatermark);
}