/** * Acquire LogSegment's monitor so that there is no concurrent loading. */ synchronized LogEntryProto loadCache(LogRecord record) throws RaftLogIOException { LogEntryProto entry = entryCache.get(record.getTermIndex()); if (entry != null) { return entry; } try { entry = cacheLoader.load(record); hasEntryCache = true; return entry; } catch (Exception e) { throw new RaftLogIOException(e); } }
if (currentLast != null) { Preconditions.assertTrue( entry.getIndex() == currentLast.getTermIndex().getIndex() + 1, "gap between entries %s and %s", entry.getIndex(), currentLast.getTermIndex().getIndex()); final LogRecord record = new LogRecord(totalSize, entry); records.add(record); if (keepEntryInCache) { entryCache.put(record.getTermIndex(), entry); configEntries.add(record.getTermIndex());
Preconditions.assertTrue(start == segment.records.get(0).getTermIndex().getIndex());
@Test public void testTruncate() throws Exception { final long term = 1; final long start = 1000; LogSegment segment = LogSegment.newOpenSegment(null, start); for (int i = 0; i < 100; i++) { LogEntryProto entry = ServerProtoUtils.toLogEntryProto( new SimpleOperation("m" + i).getLogEntryContent(), term, i + start); segment.appendToOpenSegment(entry); } // truncate an open segment (remove 1080~1099) long newSize = segment.getLogRecord(start + 80).getOffset(); segment.truncate(start + 80); Assert.assertEquals(80, segment.numOfEntries()); checkLogSegment(segment, start, start + 79, false, newSize, term); // truncate a closed segment (remove 1050~1079) newSize = segment.getLogRecord(start + 50).getOffset(); segment.truncate(start + 50); Assert.assertEquals(50, segment.numOfEntries()); checkLogSegment(segment, start, start + 49, false, newSize, term); // truncate all the remaining entries segment.truncate(start); Assert.assertEquals(0, segment.numOfEntries()); checkLogSegment(segment, start, start - 1, false, SegmentedRaftLogFormat.getHeaderLength(), term); }
if (currentLast != null) { Preconditions.assertTrue( entry.getIndex() == currentLast.getTermIndex().getIndex() + 1, "gap between entries %s and %s", entry.getIndex(), currentLast.getTermIndex().getIndex()); final LogRecord record = new LogRecord(totalSize, entry); records.add(record); if (keepEntryInCache) { entryCache.put(record.getTermIndex(), entry); configEntries.add(record.getTermIndex());
static LogSegment loadSegment(RaftStorage storage, File file, long start, long end, boolean isOpen, boolean keepEntryInCache, Consumer<LogEntryProto> logConsumer) throws IOException { final LogSegment segment = isOpen ? LogSegment.newOpenSegment(storage, start) : LogSegment.newCloseSegment(storage, start, end); readSegmentFile(file, start, end, isOpen, entry -> { segment.append(keepEntryInCache | isOpen, entry); if (logConsumer != null) { logConsumer.accept(entry); } }); // truncate padding if necessary if (file.length() > segment.getTotalSize()) { FileUtils.truncateFile(file, segment.getTotalSize()); } Preconditions.assertTrue(start == segment.getStartIndex()); if (!segment.records.isEmpty()) { Preconditions.assertTrue(start == segment.records.get(0).getTermIndex().getIndex()); } if (!isOpen) { Preconditions.assertTrue(segment.getEndIndex() == end); } return segment; }
@Test public void testTruncate() throws Exception { final long term = 1; final long start = 1000; LogSegment segment = LogSegment.newOpenSegment(null, start); for (int i = 0; i < 100; i++) { LogEntryProto entry = ProtoUtils.toLogEntryProto( new SimpleOperation("m" + i).getLogEntryContent(), term, i + start, clientId, callId); segment.appendToOpenSegment(entry); } // truncate an open segment (remove 1080~1099) long newSize = segment.getLogRecord(start + 80).getOffset(); segment.truncate(start + 80); Assert.assertEquals(80, segment.numOfEntries()); checkLogSegment(segment, start, start + 79, false, newSize, term); // truncate a closed segment (remove 1050~1079) newSize = segment.getLogRecord(start + 50).getOffset(); segment.truncate(start + 50); Assert.assertEquals(50, segment.numOfEntries()); checkLogSegment(segment, start, start + 49, false, newSize, term); // truncate all the remaining entries segment.truncate(start); Assert.assertEquals(0, segment.numOfEntries()); checkLogSegment(segment, start, start - 1, false, SegmentedRaftLog.HEADER_BYTES.length, term); }
static void checkLogSegment(LogSegment segment, long start, long end, boolean isOpen, long totalSize, long term) throws Exception { Assert.assertEquals(start, segment.getStartIndex()); Assert.assertEquals(end, segment.getEndIndex()); Assert.assertEquals(isOpen, segment.isOpen()); Assert.assertEquals(totalSize, segment.getTotalSize()); long offset = SegmentedRaftLogFormat.getHeaderLength(); for (long i = start; i <= end; i++) { LogSegment.LogRecord record = segment.getLogRecord(i); LogRecordWithEntry lre = segment.getEntryWithoutLoading(i); Assert.assertEquals(i, lre.getRecord().getTermIndex().getIndex()); Assert.assertEquals(term, lre.getRecord().getTermIndex().getTerm()); Assert.assertEquals(offset, record.getOffset()); LogEntryProto entry = lre.hasEntry() ? lre.getEntry() : segment.loadCache(lre.getRecord()); offset += getEntrySize(entry); } }
@Override public TermIndex next() { LogRecord record; if (currentSegment == null || (record = currentSegment.getLogRecord(nextIndex)) == null) { throw new NoSuchElementException(); } if (++nextIndex > currentSegment.getEndIndex()) { if (currentSegment != openSegment) { segmentIndex++; currentSegment = segmentIndex == closedSegments.size() ? openSegment : closedSegments.get(segmentIndex); } } return record.getTermIndex(); } }
private void testIterator(long startIndex) throws IOException { Iterator<TermIndex> iterator = cache.iterator(startIndex); TermIndex prev = null; while (iterator.hasNext()) { TermIndex termIndex = iterator.next(); Assert.assertEquals(cache.getLogRecord(termIndex.getIndex()).getTermIndex(), termIndex); if (prev != null) { Assert.assertEquals(prev.getIndex() + 1, termIndex.getIndex()); } prev = termIndex; } if (startIndex <= cache.getEndIndex()) { Assert.assertNotNull(prev); Assert.assertEquals(cache.getEndIndex(), prev.getIndex()); } }
private void checkLogSegment(LogSegment segment, long start, long end, boolean isOpen, long totalSize, long term) throws Exception { Assert.assertEquals(start, segment.getStartIndex()); Assert.assertEquals(end, segment.getEndIndex()); Assert.assertEquals(isOpen, segment.isOpen()); Assert.assertEquals(totalSize, segment.getTotalSize()); long offset = SegmentedRaftLog.HEADER_BYTES.length; for (long i = start; i <= end; i++) { LogSegment.LogRecord record = segment.getLogRecord(i); LogRecordWithEntry lre = segment.getEntryWithoutLoading(i); Assert.assertEquals(i, lre.getRecord().getTermIndex().getIndex()); Assert.assertEquals(term, lre.getRecord().getTermIndex().getTerm()); Assert.assertEquals(offset, record.getOffset()); LogEntryProto entry = lre.hasEntry() ? lre.getEntry() : segment.loadCache(lre.getRecord()); offset += getEntrySize(entry); } }
@Override public TermIndex next() { LogRecord record; if (currentSegment == null || (record = currentSegment.getLogRecord(nextIndex)) == null) { throw new NoSuchElementException(); } if (++nextIndex > currentSegment.getEndIndex()) { if (currentSegment != openSegment) { segmentIndex++; currentSegment = segmentIndex == closedSegments.size() ? openSegment : closedSegments.get(segmentIndex); } } return record.getTermIndex(); } }
private void testIterator(long startIndex) throws IOException { Iterator<TermIndex> iterator = cache.iterator(startIndex); TermIndex prev = null; while (iterator.hasNext()) { TermIndex termIndex = iterator.next(); Assert.assertEquals(cache.getLogRecord(termIndex.getIndex()).getTermIndex(), termIndex); if (prev != null) { Assert.assertEquals(prev.getIndex() + 1, termIndex.getIndex()); } prev = termIndex; } if (startIndex <= cache.getEndIndex()) { Assert.assertNotNull(prev); Assert.assertEquals(cache.getEndIndex(), prev.getIndex()); } }
/** * Acquire LogSegment's monitor so that there is no concurrent loading. */ synchronized LogEntryProto loadCache(LogRecord record) throws RaftLogIOException { LogEntryProto entry = entryCache.get(record.getTermIndex()); if (entry != null) { return entry; } try { entry = cacheLoader.load(record); hasEntryCache = true; return entry; } catch (Exception e) { throw new RaftLogIOException(e); } }
/** * Remove records from the given index (inclusive) */ void truncate(long fromIndex) { Preconditions.assertTrue(fromIndex >= startIndex && fromIndex <= endIndex); LogRecord record = records.get(Math.toIntExact(fromIndex - startIndex)); for (long index = endIndex; index >= fromIndex; index--) { LogRecord removed = records.remove(Math.toIntExact(index - startIndex)); entryCache.remove(removed.getTermIndex()); configEntries.remove(removed.getTermIndex()); } totalSize = record.offset; isOpen = false; this.endIndex = fromIndex - 1; }
/** * Remove records from the given index (inclusive) */ void truncate(long fromIndex) { Preconditions.assertTrue(fromIndex >= startIndex && fromIndex <= endIndex); LogRecord record = records.get(Math.toIntExact(fromIndex - startIndex)); for (long index = endIndex; index >= fromIndex; index--) { LogRecord removed = records.remove(Math.toIntExact(index - startIndex)); entryCache.remove(removed.getTermIndex()); configEntries.remove(removed.getTermIndex()); } totalSize = record.offset; isOpen = false; this.endIndex = fromIndex - 1; }
TermIndex getLastTermIndex() { LogRecord last = getLastRecord(); return last == null ? null : last.getTermIndex(); }
@Override public TermIndex getTermIndex(long index) { checkLogState(); try(AutoCloseableLock readLock = readLock()) { LogRecord record = cache.getLogRecord(index); return record != null ? record.getTermIndex() : null; } }
LogRecordWithEntry getEntryWithoutLoading(long index) { LogRecord record = getLogRecord(index); if (record == null) { return null; } return new LogRecordWithEntry(record, entryCache.get(record.getTermIndex())); }
private static void getFromSegment(LogSegment segment, long startIndex, TermIndex[] entries, int offset, int size) { long endIndex = segment.getEndIndex(); endIndex = Math.min(endIndex, startIndex + size - 1); int index = offset; for (long i = startIndex; i <= endIndex; i++) { LogRecord r = segment.getLogRecord(i); entries[index++] = r == null ? null : r.getTermIndex(); } }