private long getConsumerMaxAckedPullLogSequence(final String subject, final String group, final String consumerId) { final ConsumerProgress consumer = getConsumerProgress(subject, group, consumerId); if (consumer == null) { return -1; } else { return consumer.getAck(); } }
private long computeAckLag(final ConsumerGroupProgress progress) { if (progress == null) { return 0; } final Map<String, ConsumerProgress> consumers = progress.getConsumers(); if (consumers == null || consumers.isEmpty()) { return progress.getPull(); } long totalAckLag = 0; for (final ConsumerProgress consumer : consumers.values()) { // TODO(keli.wang): check if pull - ack is the right lag final long ackLag = consumer.getPull() - consumer.getAck(); totalAckLag += ackLag; } return totalAckLag; } }
private void putConsumer(final ConcurrentMap<String, ConcurrentMap<ConsumerGroup, ConsumerSequence>> result, final ConsumerProgress consumer) { final String consumerId = consumer.getConsumerId(); ConcurrentMap<ConsumerGroup, ConsumerSequence> consumerSequences = result.get(consumerId); if (consumerSequences == null) { consumerSequences = new ConcurrentHashMap<>(); result.putIfAbsent(consumerId, consumerSequences); } final ConsumerSequence consumerSequence = new ConsumerSequence(consumer.getPull(), consumer.getAck()); final ConsumerGroup consumerGroup = new ConsumerGroup(consumer.getSubject(), consumer.getGroup()); consumerSequences.putIfAbsent(consumerGroup, consumerSequence); }
@Override public byte[] toBytes(final ActionCheckpoint state) { final StringBuilder data = new StringBuilder(); data.append(VERSION_V3).append(NEWLINE); data.append(state.getOffset()).append(NEWLINE); final Table<String, String, ConsumerGroupProgress> progresses = state.getProgresses(); for (final String subject : progresses.rowKeySet()) { final Map<String, ConsumerGroupProgress> groups = progresses.row(subject); data.append(SLASH_JOINER.join(subject, groups.size())).append(NEWLINE); for (final String group : groups.keySet()) { final ConsumerGroupProgress progress = groups.get(group); final Map<String, ConsumerProgress> consumers = progress.getConsumers(); final int consumerCount = consumers == null ? 0 : consumers.size(); data.append(SLASH_JOINER.join(group, boolean2Short(progress.isBroadcast()), progress.getPull(), consumerCount)).append(NEWLINE); if (consumerCount <= 0) { continue; } consumers.values().forEach(consumer -> { data.append(SLASH_JOINER.join(consumer.getConsumerId(), consumer.getPull(), consumer.getAck())).append(NEWLINE); }); } } return data.toString().getBytes(Charsets.UTF_8); }
public void clean(Collection<ConsumerGroupProgress> progresses) { for (ConsumerGroupProgress progress : progresses) { final Map<String, ConsumerProgress> consumers = progress.getConsumers(); if (consumers == null || consumers.isEmpty()) { continue; } for (final ConsumerProgress consumer : consumers.values()) { PullLog pullLog = get(consumer.getSubject(), consumer.getGroup(), consumer.getConsumerId()); if (pullLog == null) continue; pullLog.clean(consumer.getAck()); } } }
public ConsumerProgress(ConsumerProgress progress) { this.subject = progress.getSubject(); this.group = progress.getGroup(); this.consumerId = progress.getConsumerId(); this.pull = progress.getPull(); this.ack = progress.getAck(); }