@Override public void destroyPullLog(String subject, String group, String consumerId) { if (pullLogManager.destroy(subject, group, consumerId)) { checkpointManager.removeConsumerProgress(subject, group, consumerId); } }
@Override public Table<String, String, PullLog> allPullLogs() { return pullLogManager.getLogs(); }
@Override public long getMessageSequenceByPullLog(String subject, String group, String consumerId, long pullLogSequence) { final PullLog log = pullLogManager.get(subject, group, consumerId); if (log == null) { return -1; } return log.getMessageSequence(pullLogSequence); }
public PullLogManager(final StorageConfig config) { this.config = config; this.logs = HashBasedTable.create(); loadPullLogs(); }
@Override public List<PutMessageResult> putPullLogs(String subject, String group, String consumerId, List<PullLogMessage> messages) { final PullLog pullLog = pullLogManager.getOrCreate(subject, group, consumerId); return pullLog.putPullLogMessages(messages); }
private void loadPullLogs() { final File pullLogsRoot = new File(config.getPullLogStorePath()); final File[] consumerIdDirs = pullLogsRoot.listFiles(); if (consumerIdDirs != null) { for (final File consumerIdDir : consumerIdDirs) { if (!consumerIdDir.isDirectory()) { continue; } loadPullLogsByConsumerId(consumerIdDir); } } }
private synchronized void submitFlushTask() { counter.set(0); latestFlushTime = System.currentTimeMillis(); final Snapshot<ActionCheckpoint> snapshot = checkpointManager.createActionCheckpointSnapshot(); flushExecutor.submit(() -> { final long start = System.currentTimeMillis(); try { pullLogManager.flush(); checkpointManager.saveActionCheckpointSnapshot(snapshot); } catch (Exception e) { QMon.pullLogFlusherFlushFailedCountInc(); LOG.error("flush pull log failed.", e); } finally { QMon.pullLogFlusherElapsedPerExecute(System.currentTimeMillis() - start); } }); }
@Override public void run() { try { messageLog.clean(); consumerLogManager.clean(); pullLogManager.clean(allConsumerGroupProgresses()); actionLog.clean(); } catch (Throwable e) { LOG.error("log cleaner caught exception.", e); } } }
public DefaultStorage(final BrokerRole role, final StorageConfig config, final CheckpointLoader loader) { this.config = config; this.consumerLogManager = new ConsumerLogManager(config); this.messageLog = new MessageLog(config, consumerLogManager); this.pullLogManager = new PullLogManager(config); this.actionLog = new ActionLog(config); this.checkpointManager = new CheckpointManager(role, config, loader); this.checkpointManager.fixOldVersionCheckpointIfShould(consumerLogManager, pullLogManager); // must init after offset manager created this.consumeQueueManager = new ConsumeQueueManager(this); this.pullLogFlusher = new PullLogFlusher(config, checkpointManager, pullLogManager); this.actionEventBus = new FixedExecOrderEventBus(); this.actionEventBus.subscribe(ActionEvent.class, new PullLogBuilder(this)); this.actionEventBus.subscribe(ActionEvent.class, new MaxSequencesUpdater(checkpointManager)); this.actionEventBus.subscribe(ActionEvent.class, pullLogFlusher); this.actionLogIterateService = new ActionLogIterateService(actionLog, checkpointManager, actionEventBus); this.consumerLogFlusher = new ConsumerLogFlusher(config, checkpointManager, consumerLogManager); this.messageEventBus = new FixedExecOrderEventBus(); this.messageEventBus.subscribe(MessageLogMeta.class, new BuildConsumerLogEventListener(consumerLogManager)); this.messageEventBus.subscribe(MessageLogMeta.class, consumerLogFlusher); this.messageLogIterateService = new MessageLogIterateService(messageLog, checkpointManager, messageEventBus); this.logCleanerExecutor = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("log-cleaner-%d").build()); this.messageLogFlushService = new PeriodicFlushService(new MessageLogFlushProvider()); this.actionLogFlushService = new PeriodicFlushService(new ActionLogFlushProvider()); }
public void clean(Collection<ConsumerGroupProgress> progresses) { for (ConsumerGroupProgress progress : progresses) { final Map<String, ConsumerProgress> consumers = progress.getConsumers(); if (consumers == null || consumers.isEmpty()) { continue; } for (final ConsumerProgress consumer : consumers.values()) { PullLog pullLog = get(consumer.getSubject(), consumer.getGroup(), consumer.getConsumerId()); if (pullLog == null) continue; pullLog.clean(consumer.getAck()); } } }
private void fixActionCheckpoint(PullLogManager manager) { final Table<String, String, PullLog> allLogs = manager.getLogs(); final Table<String, String, ConsumerGroupProgress> progresses = actionCheckpoint.getProgresses(); progresses.values().forEach(progress -> { final String subject = progress.getSubject(); final String group = progress.getGroup(); final String groupAndSubject = GroupAndSubject.groupAndSubject(subject, group); final Map<String, ConsumerProgress> consumers = progress.getConsumers(); final Map<String, PullLog> logs = allLogs.column(groupAndSubject); logs.forEach((consumerId, log) -> { final long pull = log.getMaxOffset() - 1; final ConsumerProgress consumer = consumers.get(consumerId); if (consumer != null) { consumer.setPull(pull); } else { consumers.put(consumerId, new ConsumerProgress(subject, group, consumerId, pull, -1)); } }); if (consumers.size() == 1) { consumers.values().forEach(consumer -> { if (consumer.getPull() < 0) { progress.setBroadcast(true); consumer.setPull(progress.getPull()); } }); } }); }