private long initialIterateFrom(final ActionLog log, final CheckpointManager checkpointManager) { final long checkpointOffset = checkpointManager.getActionCheckpointOffset(); final long maxOffset = log.getMaxOffset(); if (checkpointOffset <= 0) { return maxOffset; } if (checkpointOffset > maxOffset) { return maxOffset; } return checkpointOffset; }
private long initialMessageIterateFrom(final MessageLog log, final CheckpointManager checkpointManager) { if (checkpointManager.getMessageCheckpointOffset() <= 0) { return log.getMaxOffset(); } if (checkpointManager.getMessageCheckpointOffset() > log.getMaxOffset()) { return log.getMaxOffset(); } return checkpointManager.getMessageCheckpointOffset(); }
@Override public Collection<ConsumerGroupProgress> allConsumerGroupProgresses() { return checkpointManager.allConsumerGroupProgresses(); }
@Override public CompletableFuture<Datagram> processRequest(ChannelHandlerContext ctx, RemotingCommand request) { final byte[] message = checkpointManager.dumpMessageCheckpoint(); final byte[] action = checkpointManager.dumpActionCheckpoint(); final CheckpointPayloadHolder payloadHolder = new CheckpointPayloadHolder(message, action); final Datagram datagram = RemotingBuilder.buildResponseDatagram(CommandCode.SUCCESS, request.getHeader(), payloadHolder); return CompletableFuture.completedFuture(datagram); }
private void waitUntilSyncDone() { final CheckpointManager checkpointManager = storage.getCheckpointManager(); final long messageCheckpointOffset = checkpointManager.getMessageCheckpointOffset(); final long actionCheckpointOffset = checkpointManager.getActionCheckpointOffset(); while (true) { final long maxMessageOffset = storage.getMaxMessageOffset(); final long maxActionLogOffset = storage.getMaxActionLogOffset(); if (maxMessageOffset >= messageCheckpointOffset && maxActionLogOffset >= actionCheckpointOffset) { return; } LOG.info("waiting log sync done ..."); try { TimeUnit.SECONDS.sleep(2); } catch (InterruptedException ignore) { LOG.debug("sleep interrupted"); } } }
private synchronized void submitFlushTask() { counter.set(0); latestFlushTime = System.currentTimeMillis(); final Snapshot<ActionCheckpoint> snapshot = checkpointManager.createActionCheckpointSnapshot(); flushExecutor.submit(() -> { final long start = System.currentTimeMillis(); try { pullLogManager.flush(); checkpointManager.saveActionCheckpointSnapshot(snapshot); } catch (Exception e) { QMon.pullLogFlusherFlushFailedCountInc(); LOG.error("flush pull log failed.", e); } finally { QMon.pullLogFlusherElapsedPerExecute(System.currentTimeMillis() - start); } }); }
private synchronized void submitFlushTask() { counter.set(0); latestFlushTime = System.currentTimeMillis(); final Snapshot<MessageCheckpoint> snapshot = checkpointManager.createMessageCheckpointSnapshot(); flushExecutor.submit(() -> { final long start = System.currentTimeMillis(); try { consumerLogManager.flush(); checkpointManager.saveMessageCheckpointSnapshot(snapshot); } catch (Exception e) { QMon.consumerLogFlusherFlushFailedCountInc(); LOG.error("flush consumer log failed. offset: {}", snapshot.getVersion(), e); } finally { QMon.consumerLogFlusherElapsedPerExecute(System.currentTimeMillis() - start); } }); }
public DefaultStorage(final BrokerRole role, final StorageConfig config, final CheckpointLoader loader) { this.config = config; this.consumerLogManager = new ConsumerLogManager(config); this.messageLog = new MessageLog(config, consumerLogManager); this.pullLogManager = new PullLogManager(config); this.actionLog = new ActionLog(config); this.checkpointManager = new CheckpointManager(role, config, loader); this.checkpointManager.fixOldVersionCheckpointIfShould(consumerLogManager, pullLogManager); // must init after offset manager created this.consumeQueueManager = new ConsumeQueueManager(this); this.pullLogFlusher = new PullLogFlusher(config, checkpointManager, pullLogManager); this.actionEventBus = new FixedExecOrderEventBus(); this.actionEventBus.subscribe(ActionEvent.class, new PullLogBuilder(this)); this.actionEventBus.subscribe(ActionEvent.class, new MaxSequencesUpdater(checkpointManager)); this.actionEventBus.subscribe(ActionEvent.class, pullLogFlusher); this.actionLogIterateService = new ActionLogIterateService(actionLog, checkpointManager, actionEventBus); this.consumerLogFlusher = new ConsumerLogFlusher(config, checkpointManager, consumerLogManager); this.messageEventBus = new FixedExecOrderEventBus(); this.messageEventBus.subscribe(MessageLogMeta.class, new BuildConsumerLogEventListener(consumerLogManager)); this.messageEventBus.subscribe(MessageLogMeta.class, consumerLogFlusher); this.messageLogIterateService = new MessageLogIterateService(messageLog, checkpointManager, messageEventBus); this.logCleanerExecutor = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("log-cleaner-%d").build()); this.messageLogFlushService = new PeriodicFlushService(new MessageLogFlushProvider()); this.actionLogFlushService = new PeriodicFlushService(new ActionLogFlushProvider()); }
Snapshot<MessageCheckpoint> createMessageCheckpointSnapshot() { final MessageCheckpoint checkpoint = duplicateMessageCheckpoint(); return new Snapshot<>(checkpoint.getOffset(), checkpoint); }
Snapshot<ActionCheckpoint> createActionCheckpointSnapshot() { final ActionCheckpoint checkpoint = duplicateActionCheckpoint(); return new Snapshot<>(checkpoint.getOffset(), checkpoint); }