private long nextSegmentBaseOffset() { final LogSegment segment = latestSegment(); if (segment == null) { return 0; } else { return segment.getBaseOffset() + fileSize; } }
public void deleteSegmentsBeforeOffset(final long offset) { if (offset == -1) return; Predicate<LogSegment> predicate = segment -> segment.getBaseOffset() < offset; deleteSegments(predicate, null); }
public long getMinOffset() { final LogSegment segment = firstSegment(); if (segment == null) { return 0; } return segment.getBaseOffset(); }
static String offsetFileNameForSegment(final LogSegment segment) { return offsetFileNameOf(segment.getBaseOffset()); }
private void loadLogs() { LOG.info("Loading logs."); try { final File[] files = logDir.listFiles(); if (files == null) return; for (final File file : files) { if (file.getName().startsWith(".")) { continue; } try { final LogSegment segment = new LogSegment(file, fileSize); segment.setWrotePosition(fileSize); segments.put(segment.getBaseOffset(), segment); LOG.info("Load {} success.", file.getAbsolutePath()); } catch (IOException e) { LOG.error("Load {} failed.", file.getAbsolutePath()); } } } finally { LOG.info("Load logs done."); } }
public SegmentBuffer selectSegmentBuffer(final int pos) { final int wrotePosition = getWrotePosition(); if (pos < wrotePosition && pos >= 0) { final ByteBuffer buffer = mappedByteBuffer.slice(); buffer.position(pos); final ByteBuffer bufferNew = buffer.slice(); final int size = wrotePosition - pos; bufferNew.limit(size); return new SegmentBuffer(getBaseOffset() + pos, bufferNew, size, this); } return null; }
public SegmentBuffer selectSegmentBuffer(final int pos, final int size) { final int wrotePosition = getWrotePosition(); if ((pos + size) > wrotePosition) { return null; } final ByteBuffer buffer = mappedByteBuffer.slice(); buffer.position(pos); final ByteBuffer bufferNew = buffer.slice(); bufferNew.limit(size); return new SegmentBuffer(getBaseOffset() + pos, bufferNew, size, this); }
public long getMaxOffset() { final LogSegment segment = latestSegment(); if (segment == null) { return 0; } return segment.getBaseOffset() + segment.getWrotePosition(); }
public boolean flush() { boolean result = true; final LogSegment segment = locateSegment(flushedOffset); if (segment != null) { final int offset = segment.flush(); final long where = segment.getBaseOffset() + offset; result = where == this.flushedOffset; this.flushedOffset = where; } return result; }
private boolean acquireSegmentLock(final SegmentBuffer result) { if (BrokerConfig.getBrokerRole() != BrokerRole.MASTER) { return true; } final LogSegment segment = result.getLogSegment(); if (segment == null) { return true; } if (currentSegment != null && currentSegment.getBaseOffset() == segment.getBaseOffset()) { return true; } if (currentSegment != null) { final boolean release = currentSegment.release(); LOG.info("release segment lock for {} while sync log, release result: {}", currentSegment, release); currentSegment = null; } final boolean retain = segment.retain(); LOG.info("acquire segment lock for {} while sync log, retain result: {}", segment, retain); if (retain) { currentSegment = segment; } return retain; }
visitedBufferSize.addAndGet(wroteBytes); return Optional.of(new MessageLogMeta(new String(subjectBytes, StandardCharsets.UTF_8), sequence, wroteOffset, wroteBytes, headerSize, currentSegment.getBaseOffset()));
private void fillPreBlank(final LogSegment segment, final long untilWhere) { final ConsumerLogMessage blankMessage = new ConsumerLogMessage(0, 0, Integer.MAX_VALUE, (short) 0); final long startOffset = segment.getBaseOffset() + segment.getWrotePosition(); for (long i = startOffset; i < untilWhere; i += CONSUMER_LOG_UNIT_BYTES) { segment.append(blankMessage, consumerLogAppender); } }
public <T, R> AppendMessageResult<R> append(final T message, final MessageAppender<T, R> appender) { final int currentPos = wrotePosition.get(); if (currentPos > fileSize) { return new AppendMessageResult<>(AppendMessageStatus.UNKNOWN_ERROR); } if (currentPos == fileSize) { return new AppendMessageResult<>(AppendMessageStatus.END_OF_FILE); } final ByteBuffer buffer = mappedByteBuffer.slice(); buffer.position(currentPos); final AppendMessageResult<R> result = appender.doAppend(getBaseOffset(), buffer, fileSize - currentPos, message); this.wrotePosition.addAndGet(result.getWroteBytes()); return result; }
private void fillPreBlank(final LogSegment segment, final long untilWhere) { final PullLogMessage blankMessage = new PullLogMessage(0, -1); final long startOffset = segment.getBaseOffset() + segment.getWrotePosition(); for (long i = startOffset; i < untilWhere; i += PULL_LOG_UNIT_BYTES) { segment.append(blankMessage, messageAppender); } }
offset = segment.getBaseOffset(); final LogSegmentValidator.ValidateResult result = segmentValidator.validate(segment); offset += result.getValidatedSize(); final long maxOffset = latestSegment().getBaseOffset() + latestSegment().getFileSize(); final int relativeOffset = (int) (offset % fileSize); final LogSegment segment = locateSegment(offset);
public LogSegment allocOrResetSegments(final long expectedOffset) { final long baseOffset = computeBaseOffset(expectedOffset); if (segments.isEmpty()) { return allocSegment(baseOffset); } if (nextSegmentBaseOffset() == baseOffset && latestSegment().isFull()) { return allocSegment(baseOffset); } LOG.warn("All segments are too old, need to delete all segment now. Current base offset: {}, expect base offset: {}", latestSegment().getBaseOffset(), baseOffset); deleteAllSegments(); return allocSegment(baseOffset); }