@Override public PutMessageResult putAction(Action action) { return actionLog.addAction(action); }
@Override public boolean appendActionLogData(long startOffset, ByteBuffer data) { return actionLog.appendData(startOffset, data); }
@Override public void flush() { actionLog.flush(); } }
public void blockUntilReplayDone() { LOG.info("replay action log initial lag: {}; min: {}, max: {}, from: {}", replayActionLogLag(), log.getMinOffset(), log.getMaxOffset(), iterateFrom.longValue()); while (replayActionLogLag() > 0) { LOG.info("waiting replay action log ..."); try { TimeUnit.SECONDS.sleep(1); } catch (InterruptedException e) { LOG.warn("block until replay done interrupted", e); } } }
@Override public long getMaxActionLogOffset() { return actionLog.getMaxOffset(); }
public long getMinActionLogOffset() { return actionLog.getMinOffset(); }
@Override public SegmentBuffer getActionLogData(long offset) { return actionLog.getMessageData(offset); }
@Override public void run() { try { messageLog.clean(); consumerLogManager.clean(); pullLogManager.clean(allConsumerGroupProgresses()); actionLog.clean(); } catch (Throwable e) { LOG.error("log cleaner caught exception.", e); } } }
public boolean appendData(final long startOffset, final ByteBuffer data) { LogSegment segment = logManager.locateSegment(startOffset); if (segment == null) { segment = logManager.allocOrResetSegments(startOffset); fillPreBlank(segment, startOffset); } return segment.appendData(data); }
private void processLog() { long startOffset = iterateFrom.longValue(); final ActionLogVisitor visitor = log.newVisitor(iterateFrom.longValue()); if (startOffset != visitor.getStartOffset()) { iterateFrom.reset(); iterateFrom.add(visitor.getStartOffset()); } while (true) { final Optional<Action> action = visitor.nextAction(); if (action == null) { break; } action.ifPresent(act -> dispatcher.post(new ActionEvent(iterateFrom.longValue() + visitor.visitedBufferSize(), act))); } iterateFrom.add(visitor.visitedBufferSize()); try { TimeUnit.MILLISECONDS.sleep(5); } catch (InterruptedException e) { LOG.warn("action log dispatcher sleep interrupted"); } } }
public DefaultStorage(final BrokerRole role, final StorageConfig config, final CheckpointLoader loader) { this.config = config; this.consumerLogManager = new ConsumerLogManager(config); this.messageLog = new MessageLog(config, consumerLogManager); this.pullLogManager = new PullLogManager(config); this.actionLog = new ActionLog(config); this.checkpointManager = new CheckpointManager(role, config, loader); this.checkpointManager.fixOldVersionCheckpointIfShould(consumerLogManager, pullLogManager); // must init after offset manager created this.consumeQueueManager = new ConsumeQueueManager(this); this.pullLogFlusher = new PullLogFlusher(config, checkpointManager, pullLogManager); this.actionEventBus = new FixedExecOrderEventBus(); this.actionEventBus.subscribe(ActionEvent.class, new PullLogBuilder(this)); this.actionEventBus.subscribe(ActionEvent.class, new MaxSequencesUpdater(checkpointManager)); this.actionEventBus.subscribe(ActionEvent.class, pullLogFlusher); this.actionLogIterateService = new ActionLogIterateService(actionLog, checkpointManager, actionEventBus); this.consumerLogFlusher = new ConsumerLogFlusher(config, checkpointManager, consumerLogManager); this.messageEventBus = new FixedExecOrderEventBus(); this.messageEventBus.subscribe(MessageLogMeta.class, new BuildConsumerLogEventListener(consumerLogManager)); this.messageEventBus.subscribe(MessageLogMeta.class, consumerLogFlusher); this.messageLogIterateService = new MessageLogIterateService(messageLog, checkpointManager, messageEventBus); this.logCleanerExecutor = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("log-cleaner-%d").build()); this.messageLogFlushService = new PeriodicFlushService(new MessageLogFlushProvider()); this.actionLogFlushService = new PeriodicFlushService(new ActionLogFlushProvider()); }
private long initialIterateFrom(final ActionLog log, final CheckpointManager checkpointManager) { final long checkpointOffset = checkpointManager.getActionCheckpointOffset(); final long maxOffset = log.getMaxOffset(); if (checkpointOffset <= 0) { return maxOffset; } if (checkpointOffset > maxOffset) { return maxOffset; } return checkpointOffset; }
private long replayActionLogLag() { return log.getMaxOffset() - iterateFrom.longValue(); }
public synchronized PutMessageResult addAction(final Action action) { final AppendMessageResult<MessageSequence> result; LogSegment segment = logManager.latestSegment(); if (segment == null) { segment = logManager.allocNextSegment(); } if (segment == null) { return new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null); } result = segment.append(action, actionAppender); switch (result.getStatus()) { case SUCCESS: break; case END_OF_FILE: if (logManager.allocNextSegment() == null) { return new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null); } return addAction(action); case MESSAGE_SIZE_EXCEEDED: return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result); default: return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result); } return new PutMessageResult(PutMessageStatus.SUCCESS, result); }