public IterateOffsetManager(String checkpointStorePath, FlushHook hook) { this.offsetCheckpointStore = new CheckpointStore<>(checkpointStorePath, ITERATE_OFFSET_FILE, new IterateCheckpointSerde()); Long offset = this.offsetCheckpointStore.loadCheckpoint(); if (null != offset) { this.iterateOffset = offset; } this.flushHook = hook; }
@Override public void flush() { flushHook.beforeFlush(); offsetCheckpointStore.saveCheckpoint(iterateOffset); } };
public CheckpointStore(final String storePath, final String filename, final Serde<T> serde) { this.storePath = storePath; this.filename = filename; this.serde = serde; ensureDir(storePath); }
void createOffsetFileFor(long baseOffset, Map<String, Long> offsets) { final String fileName = StoreUtils.offsetFileNameOf(baseOffset); final CheckpointStore<Map<String, Long>> offsetStore = new CheckpointStore<>(config.getMessageLogStorePath(), fileName, new ConsumerLogMinOffsetSerde()); offsetStore.saveCheckpoint(offsets); }
final File tmp = tmpFile(); try { Files.write(data, tmp); final File checkpointFile = checkpointFile(); if (checkpointFile.exists()) { final File backupFile = backupFile();
public T loadCheckpoint() { final File checkpointFile = checkpointFile(); final File backupFile = backupFile(); if (!checkpointFile.exists() && !backupFile.exists()) { LOG.warn("Checkpoint file and backup file does not exist, return null for now"); return null; } try { final byte[] data = Files.toByteArray(checkpointFile); if (data != null && data.length == 0) { return null; } return serde.fromBytes(data); } catch (IOException e) { LOG.error("Load checkpoint file failed. Try load backup checkpoint file instead.", e); } try { return serde.fromBytes(Files.toByteArray(backupFile)); } catch (IOException e) { LOG.error("Load backup checkpoint file failed.", e); } throw new RuntimeException("Load checkpoint failed. filename=" + filename); }
void adjustConsumerLogMinOffset(LogSegment firstSegment) { if (firstSegment == null) return; final String fileName = StoreUtils.offsetFileNameForSegment(firstSegment); final CheckpointStore<Map<String, Long>> offsetStore = new CheckpointStore<>(config.getMessageLogStorePath(), fileName, new ConsumerLogMinOffsetSerde()); final Map<String, Long> offsets = offsetStore.loadCheckpoint(); if (offsets == null) return; LOG.info("adjust consumer log min offset with offset file {}", fileName); for (Map.Entry<String, Long> entry : offsets.entrySet()) { final ConsumerLog log = logs.get(entry.getKey()); if (log == null) { LOG.warn("cannot find consumer log {} while adjust min offset.", entry.getKey()); } else { long adjustedMinOffset = entry.getValue() + 1; log.setMinSequence(adjustedMinOffset); } } }