@Override public void onEvent(ActionEvent event) { final long count = counter.incrementAndGet(); if (count < config.getActionCheckpointInterval()) { return; } QMon.pullLogFlusherExceedCheckpointIntervalCountInc(); submitFlushTask(); }
private void tryCleanExpiredSnapshot() { try { while (true) { if (snapshots.size() <= config.getCheckpointRetainCount()) { return; } removeOldestSnapshot(); } } catch (Throwable e) { LOG.error("try clean expired snapshot file failed.", e); } }
public void clean() { logManager.deleteExpiredSegments(config.getConsumerLogRetentionMs()); }
public void clean() { logManager.deleteExpiredSegments(config.getMessageLogRetentionMs(), segment -> { consumerLogManager.adjustConsumerLogMinOffset(logManager.firstSegment()); final String fileName = StoreUtils.offsetFileNameForSegment(segment); final String path = config.getMessageLogStorePath(); final File file = new File(path, fileName); try { if (!file.delete()) { LOG.warn("delete offset file failed. file: {}", fileName); } } catch (Exception e) { LOG.warn("delete offset file failed.. file: {}", fileName, e); } }); }
private void loadConsumerLogs() { LOG.info("Start load consumer logs"); final File root = new File(config.getConsumerLogStorePath()); final File[] consumerLogDirs = root.listFiles(); if (consumerLogDirs != null) { for (final File consumerLogDir : consumerLogDirs) { if (!consumerLogDir.isDirectory()) { continue; } final String subject = consumerLogDir.getName(); final ConsumerLog consumerLog = new ConsumerLog(config, subject); logs.put(subject, consumerLog); } } LOG.info("Load consumer logs done"); }
public void clean() { logManager.deleteExpiredSegments(config.getLogRetentionMs()); }
@Override public void onEvent(final MessageLogMeta event) { final long count = counter.incrementAndGet(); if (count < config.getMessageCheckpointInterval()) { return; } QMon.consumerLogFlusherExceedCheckpointIntervalCountInc(); submitFlushTask(); }
public ActionLog(final StorageConfig config) { this.config = config; this.logManager = new LogManager(new File(config.getActionLogStorePath()), PER_SEGMENT_FILE_SIZE, config, new ActionLogSegmentValidator()); }
public SnapshotStore(final String name, final StorageConfig config, final Serde<T> serde) { this.prefix = name + "."; this.config = config; this.storePath = new File(config.getCheckpointStorePath()); this.serde = serde; this.snapshots = new ConcurrentSkipListMap<>(); this.cleanerExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory(name + "-snapshot-cleaner")); ensureStorePath(); loadAllSnapshots(); scheduleSnapshotCleaner(); }
@Override public void start() { messageLogFlushService.start(); actionLogFlushService.start(); actionLogIterateService.start(); messageLogIterateService.start(); messageLogIterateService.blockUntilReplayDone(); actionLogIterateService.blockUntilReplayDone(); // must call this after message log replay done consumerLogManager.initConsumerLogOffset(); logCleanerExecutor.scheduleAtFixedRate( new LogCleaner(), 0, config.getLogRetentionCheckIntervalSeconds(), TimeUnit.SECONDS); }
public ConsumerLog(final StorageConfig config, final String subject) { this.config = config; this.subject = subject; this.logManager = new LogManager(new File(config.getConsumerLogStorePath(), subject), CONSUMER_LOG_SIZE, config, new ConsumerLogSegmentValidator()); }