private long computeAckLag(final ConsumerGroupProgress progress) { if (progress == null) { return 0; } final Map<String, ConsumerProgress> consumers = progress.getConsumers(); if (consumers == null || consumers.isEmpty()) { return progress.getPull(); } long totalAckLag = 0; for (final ConsumerProgress consumer : consumers.values()) { // TODO(keli.wang): check if pull - ack is the right lag final long ackLag = consumer.getPull() - consumer.getAck(); totalAckLag += ackLag; } return totalAckLag; } }
private ConsumerGroupProgress getOrCreateConsumerGroupProgress(final String subject, final String group, final boolean broadcast) { final Table<String, String, ConsumerGroupProgress> progresses = actionCheckpoint.getProgresses(); if (!progresses.contains(subject, group)) { final ConsumerGroupProgress progress = new ConsumerGroupProgress(subject, group, broadcast, -1, new HashMap<>()); progresses.put(subject, group, progress); } return progresses.get(subject, group); }
private void updateMaxPulledMessageSequence(final String subject, final String group, final boolean broadcast, final long maxSequence) { final ConsumerGroupProgress progress = getOrCreateConsumerGroupProgress(subject, group, broadcast); progress.setPull(maxSequence); }
private ActionCheckpoint duplicateActionCheckpoint() { actionCheckpointGuard.lock(); try { final Table<String, String, ConsumerGroupProgress> progresses = HashBasedTable.create(); for (final ConsumerGroupProgress progress : actionCheckpoint.getProgresses().values()) { final Map<String, ConsumerProgress> consumers = progress.getConsumers(); if (consumers == null) { continue; } final Map<String, ConsumerProgress> consumersCopy = new HashMap<>(); for (final ConsumerProgress consumer : consumers.values()) { consumersCopy.put(consumer.getConsumerId(), new ConsumerProgress(consumer)); } final String subject = progress.getSubject(); final String group = progress.getGroup(); progresses.put(subject, group, new ConsumerGroupProgress(subject, group, progress.isBroadcast(), progress.getPull(), consumersCopy)); } final long offset = actionCheckpoint.getOffset(); return new ActionCheckpoint(offset, progresses); } finally { actionCheckpointGuard.unlock(); } }
private void fixActionCheckpoint(PullLogManager manager) { final Table<String, String, PullLog> allLogs = manager.getLogs(); final Table<String, String, ConsumerGroupProgress> progresses = actionCheckpoint.getProgresses(); progresses.values().forEach(progress -> { final String subject = progress.getSubject(); final String group = progress.getGroup(); final String groupAndSubject = GroupAndSubject.groupAndSubject(subject, group); final Map<String, ConsumerProgress> consumers = progress.getConsumers(); final Map<String, PullLog> logs = allLogs.column(groupAndSubject); logs.forEach((consumerId, log) -> { final long pull = log.getMaxOffset() - 1; final ConsumerProgress consumer = consumers.get(consumerId); if (consumer != null) { consumer.setPull(pull); } else { consumers.put(consumerId, new ConsumerProgress(subject, group, consumerId, pull, -1)); } }); if (consumers.size() == 1) { consumers.values().forEach(consumer -> { if (consumer.getPull() < 0) { progress.setBroadcast(true); consumer.setPull(progress.getPull()); } }); } }); }
@Override public byte[] toBytes(final ActionCheckpoint state) { final StringBuilder data = new StringBuilder(); data.append(VERSION_V3).append(NEWLINE); data.append(state.getOffset()).append(NEWLINE); final Table<String, String, ConsumerGroupProgress> progresses = state.getProgresses(); for (final String subject : progresses.rowKeySet()) { final Map<String, ConsumerGroupProgress> groups = progresses.row(subject); data.append(SLASH_JOINER.join(subject, groups.size())).append(NEWLINE); for (final String group : groups.keySet()) { final ConsumerGroupProgress progress = groups.get(group); final Map<String, ConsumerProgress> consumers = progress.getConsumers(); final int consumerCount = consumers == null ? 0 : consumers.size(); data.append(SLASH_JOINER.join(group, boolean2Short(progress.isBroadcast()), progress.getPull(), consumerCount)).append(NEWLINE); if (consumerCount <= 0) { continue; } consumers.values().forEach(consumer -> { data.append(SLASH_JOINER.join(consumer.getConsumerId(), consumer.getPull(), consumer.getAck())).append(NEWLINE); }); } } return data.toString().getBytes(Charsets.UTF_8); }
private void initSubscribers() { final Collection<ConsumerGroupProgress> progresses = storage.allConsumerGroupProgresses(); progresses.forEach(progress -> { if (progress.isBroadcast()) { return; } progress.getConsumers().values().forEach(consumer -> { final String groupAndSubject = GroupAndSubject.groupAndSubject(consumer.getSubject(), consumer.getGroup()); addSubscriber(groupAndSubject, consumer.getConsumerId()); }); }); }
private ConsumerProgress getConsumerProgress(final String subject, final String group, final String consumerId) { final ConsumerGroupProgress progress = actionCheckpoint.getProgresses().get(subject, group); if (progress == null) { return null; } final Map<String, ConsumerProgress> consumers = progress.getConsumers(); if (consumers == null) { return null; } return consumers.get(consumerId); }
private ActionCheckpoint parseBySplitter(final LineReader reader, final Splitter splitter) throws IOException { final Table<String, String, ConsumerGroupProgress> progresses = HashBasedTable.create(); while (true) { final String subjectLine = reader.readLine(); if (Strings.isNullOrEmpty(subjectLine)) { break; } final List<String> subjectParts = splitter.splitToList(subjectLine); final String subject = subjectParts.get(0); final int groupCount = Integer.parseInt(subjectParts.get(1)); for (int i = 0; i < groupCount; i++) { final String groupLine = reader.readLine(); final List<String> groupParts = splitter.splitToList(groupLine); final String group = groupParts.get(0); final long maxPulledMessageSequence = Long.parseLong(groupParts.get(1)); final int consumerCount = Integer.parseInt(groupParts.get(2)); final ConsumerGroupProgress progress = new ConsumerGroupProgress(subject, group, false, maxPulledMessageSequence, new HashMap<>(consumerCount)); progresses.put(subject, group, progress); final Map<String, ConsumerProgress> consumers = progress.getConsumers(); for (int j = 0; j < consumerCount; j++) { final String consumerLine = reader.readLine(); final List<String> consumerParts = splitter.splitToList(consumerLine); final String consumerId = consumerParts.get(0); final long maxAckedPullLogSequence = Long.parseLong(consumerParts.get(1)); consumers.put(consumerId, new ConsumerProgress(subject, group, consumerId, -1, maxAckedPullLogSequence)); } } } return new ActionCheckpoint(true, -1, progresses); }
private long getLastMaxSequence(final String subject, final String group) { final ConsumerGroupProgress progress = storage.getConsumerGroupProgress(subject, group); if (progress == null) { return -1; } else { return progress.getPull(); } }
void removeConsumerProgress(String subject, String group, String consumerId) { final ConsumerGroupProgress progress = actionCheckpoint.getProgresses().get(subject, group); if (progress == null) { return; } final Map<String, ConsumerProgress> consumers = progress.getConsumers(); if (consumers != null) { consumers.remove(consumerId); } }
final int consumerCount = Integer.parseInt(groupParts.get(3)); final ConsumerGroupProgress progress = new ConsumerGroupProgress(subject, group, broadcast, maxPulledMessageSequence, new HashMap<>(consumerCount)); progresses.put(subject, group, progress); final Map<String, ConsumerProgress> consumers = progress.getConsumers(); for (int j = 0; j < consumerCount; j++) { final String consumerLine = reader.readLine();
long getMaxPulledMessageSequence(final String subject, final String group) { actionCheckpointGuard.lock(); try { final ConsumerGroupProgress progress = actionCheckpoint.getProgresses().get(subject, group); if (progress == null) { return -1; } return progress.getPull(); } finally { actionCheckpointGuard.unlock(); } }
private void loadFromConsumerGroupProgresses(final ConcurrentMap<String, ConcurrentMap<ConsumerGroup, ConsumerSequence>> result) { final Collection<ConsumerGroupProgress> progresses = storage.allConsumerGroupProgresses(); progresses.forEach(progress -> { final Map<String, ConsumerProgress> consumers = progress.getConsumers(); if (consumers == null || consumers.isEmpty()) { return; } consumers.values().forEach(consumer -> putConsumer(result, consumer)); }); }
private ConsumerProgress getOrCreateConsumerProgress(final String subject, final String group, final String consumerId, final boolean broadcast) { final ConsumerGroupProgress progress = getOrCreateConsumerGroupProgress(subject, group, broadcast); final Map<String, ConsumerProgress> consumers = progress.getConsumers(); if (!consumers.containsKey(consumerId)) { consumers.put(consumerId, new ConsumerProgress(subject, group, consumerId, -1, -1)); } return consumers.get(consumerId); }
public void clean(Collection<ConsumerGroupProgress> progresses) { for (ConsumerGroupProgress progress : progresses) { final Map<String, ConsumerProgress> consumers = progress.getConsumers(); if (consumers == null || consumers.isEmpty()) { continue; } for (final ConsumerProgress consumer : consumers.values()) { PullLog pullLog = get(consumer.getSubject(), consumer.getGroup(), consumer.getConsumerId()); if (pullLog == null) continue; pullLog.clean(consumer.getAck()); } } }