void adjustConsumerLogMinOffset(LogSegment firstSegment) { if (firstSegment == null) return; final String fileName = StoreUtils.offsetFileNameForSegment(firstSegment); final CheckpointStore<Map<String, Long>> offsetStore = new CheckpointStore<>(config.getMessageLogStorePath(), fileName, new ConsumerLogMinOffsetSerde()); final Map<String, Long> offsets = offsetStore.loadCheckpoint(); if (offsets == null) return; LOG.info("adjust consumer log min offset with offset file {}", fileName); for (Map.Entry<String, Long> entry : offsets.entrySet()) { final ConsumerLog log = logs.get(entry.getKey()); if (log == null) { LOG.warn("cannot find consumer log {} while adjust min offset.", entry.getKey()); } else { long adjustedMinOffset = entry.getValue() + 1; log.setMinSequence(adjustedMinOffset); } } }
public void clean() { logManager.deleteExpiredSegments(config.getMessageLogRetentionMs(), segment -> { consumerLogManager.adjustConsumerLogMinOffset(logManager.firstSegment()); final String fileName = StoreUtils.offsetFileNameForSegment(segment); final String path = config.getMessageLogStorePath(); final File file = new File(path, fileName); try { if (!file.delete()) { LOG.warn("delete offset file failed. file: {}", fileName); } } catch (Exception e) { LOG.warn("delete offset file failed.. file: {}", fileName, e); } }); }
public MessageLog(final StorageConfig config, final ConsumerLogManager consumerLogManager) { this.config = config; this.consumerLogManager = consumerLogManager; this.logManager = new LogManager(new File(config.getMessageLogStorePath()), PER_SEGMENT_FILE_SIZE, config, new MessageLogSegmentValidator()); consumerLogManager.adjustConsumerLogMinOffset(logManager.firstSegment()); }
void createOffsetFileFor(long baseOffset, Map<String, Long> offsets) { final String fileName = StoreUtils.offsetFileNameOf(baseOffset); final CheckpointStore<Map<String, Long>> offsetStore = new CheckpointStore<>(config.getMessageLogStorePath(), fileName, new ConsumerLogMinOffsetSerde()); offsetStore.saveCheckpoint(offsets); }