public void deleteExpiredSegments(final long retentionMs) { deleteExpiredSegments(retentionMs, null); }
@Override public void flush() { logManager.flush(); }
long getMaxOffset() { return logManager.getMaxOffset(); }
public LogSegment allocNextSegment() { final long nextBaseOffset = nextSegmentBaseOffset(); return allocSegment(nextBaseOffset); }
public LogSegment allocOrResetSegments(final long expectedOffset) { final long baseOffset = computeBaseOffset(expectedOffset); if (segments.isEmpty()) { return allocSegment(baseOffset); } if (nextSegmentBaseOffset() == baseOffset && latestSegment().isFull()) { return allocSegment(baseOffset); } LOG.warn("All segments are too old, need to delete all segment now. Current base offset: {}, expect base offset: {}", latestSegment().getBaseOffset(), baseOffset); deleteAllSegments(); return allocSegment(baseOffset); }
public boolean putMessageLogOffset(final long sequence, final long offset, final int size, final short headerSize) { putMessageLogOffsetLock.lock(); try { if (sequence < nextSequence()) { return true; } final long expectedOffset = sequence * CONSUMER_LOG_UNIT_BYTES; LogSegment segment = logManager.locateSegment(expectedOffset); if (segment == null) { segment = logManager.allocOrResetSegments(expectedOffset); } fillPreBlank(segment, expectedOffset); final AppendMessageResult result = segment.append(new ConsumerLogMessage(sequence, offset, size, headerSize), consumerLogAppender); switch (result.getStatus()) { case SUCCESS: break; case END_OF_FILE: logManager.allocNextSegment(); return putMessageLogOffset(sequence, offset, size, headerSize); default: return false; } } finally { putMessageLogOffsetLock.unlock(); } return true; }
boolean appendData(long startOffset, ByteBuffer buffer) { LogSegment segment = logManager.locateSegment(startOffset); if (null == segment) { segment = logManager.allocOrResetSegments(startOffset); fillPreBlank(segment, startOffset); } return segment.appendData(buffer); }
@Override public AppendMessageRecordResult append(RawMessageExtend record) { AppendMessageResult<Long> result; LogSegment segment = logManager.latestSegment(); if (null == segment) { segment = logManager.allocNextSegment(); } if (null == segment) { return new AppendMessageRecordResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null); } result = segment.append(record, messageAppender); switch (result.getStatus()) { case MESSAGE_SIZE_EXCEEDED: return new AppendMessageRecordResult(PutMessageStatus.MESSAGE_ILLEGAL, null); case END_OF_FILE: if (null == logManager.allocNextSegment()) { return new AppendMessageRecordResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null); } return append(record); case SUCCESS: return new AppendMessageRecordResult(PutMessageStatus.SUCCESS, result); default: return new AppendMessageRecordResult(PutMessageStatus.UNKNOWN_ERROR, result); } }
public SegmentBuffer getMessageData(final long offset) { final LogSegment segment = logManager.locateSegment(offset); if (segment == null) { return null; } final int pos = (int) (offset % PER_SEGMENT_FILE_SIZE); return segment.selectSegmentBuffer(pos); }
public long getMinOffset() { long computedMinSequence = logManager.getMinOffset() / CONSUMER_LOG_UNIT_BYTES; if (computedMinSequence < minSequence) { return minSequence; } else { return computedMinSequence; } }
final long maxOffset = latestSegment().getBaseOffset() + latestSegment().getFileSize(); final int relativeOffset = (int) (offset % fileSize); final LogSegment segment = locateSegment(offset); if (segment != null && maxOffset != offset) { segment.setWrotePosition(relativeOffset);
public ActionLog(final StorageConfig config) { this.config = config; this.logManager = new LogManager(new File(config.getActionLogStorePath()), PER_SEGMENT_FILE_SIZE, config, new ActionLogSegmentValidator()); }
public MessageLog(final StorageConfig config, final ConsumerLogManager consumerLogManager) { this.config = config; this.consumerLogManager = consumerLogManager; this.logManager = new LogManager(new File(config.getMessageLogStorePath()), PER_SEGMENT_FILE_SIZE, config, new MessageLogSegmentValidator()); consumerLogManager.adjustConsumerLogMinOffset(logManager.firstSegment()); }
public void clean() { logManager.deleteExpiredSegments(config.getMessageLogRetentionMs(), segment -> { consumerLogManager.adjustConsumerLogMinOffset(logManager.firstSegment()); final String fileName = StoreUtils.offsetFileNameForSegment(segment); final String path = config.getMessageLogStorePath(); final File file = new File(path, fileName); try { if (!file.delete()) { LOG.warn("delete offset file failed. file: {}", fileName); } } catch (Exception e) { LOG.warn("delete offset file failed.. file: {}", fileName, e); } }); }
private void recoverSequence() { LogSegment logSegment = logManager.latestSegment(); if (null == logSegment) { LOGGER.warn("recover sequence happened null segment"); return; } validateSequence(logSegment); }
private PutMessageResult directPutMessage(final PullLogMessage message) { final long sequence = message.getSequence(); if (sequence < getMaxOffset()) { return new PutMessageResult(PutMessageStatus.ALREADY_WRITTEN, null); } final long expectPhysicalOffset = sequence * PULL_LOG_UNIT_BYTES; LogSegment segment = logManager.locateSegment(expectPhysicalOffset); if (segment == null) { segment = logManager.allocOrResetSegments(expectPhysicalOffset); } fillPreBlank(segment, expectPhysicalOffset); final AppendMessageResult<MessageSequence> result = segment.append(message, messageAppender); switch (result.getStatus()) { case SUCCESS: break; case END_OF_FILE: logManager.allocNextSegment(); return directPutMessage(message); default: return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result); } return new PutMessageResult(PutMessageStatus.SUCCESS, result); }
public boolean appendData(final long startOffset, final ByteBuffer data) { LogSegment segment = logManager.locateSegment(startOffset); if (segment == null) { segment = logManager.allocOrResetSegments(startOffset); fillPreBlank(segment, startOffset); } return segment.appendData(data); }
public synchronized PutMessageResult addAction(final Action action) { final AppendMessageResult<MessageSequence> result; LogSegment segment = logManager.latestSegment(); if (segment == null) { segment = logManager.allocNextSegment(); } if (segment == null) { return new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null); } result = segment.append(action, actionAppender); switch (result.getStatus()) { case SUCCESS: break; case END_OF_FILE: if (logManager.allocNextSegment() == null) { return new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null); } return addAction(action); case MESSAGE_SIZE_EXCEEDED: return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result); default: return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result); } return new PutMessageResult(PutMessageStatus.SUCCESS, result); }
private SegmentBuffer selectIndexBuffer(final long startIndex) { final long startOffset = startIndex * PULL_LOG_UNIT_BYTES; final LogSegment segment = logManager.locateSegment(startOffset); if (segment == null) { return null; } else { return segment.selectSegmentBuffer((int) (startOffset % PULL_LOG_SIZE)); } }
public long getMinOffset() { return logManager.getMinOffset() / PULL_LOG_UNIT_BYTES; }