private boolean isSegmentFull(LogSegment segment, LogEntryProto entry) { if (segment.getTotalSize() >= segmentMaxSize) { return true; } else { final long entrySize = LogSegment.getEntrySize(entry); // if entry size is greater than the max segment size, write it directly // into the current segment return entrySize <= segmentMaxSize && segment.getTotalSize() + entrySize > segmentMaxSize; } }
private boolean isSegmentFull(LogSegment segment, LogEntryProto entry) { if (segment.getTotalSize() >= segmentMaxSize) { return true; } else { final long entrySize = LogSegment.getEntrySize(entry); // if entry size is greater than the max segment size, write it directly // into the current segment return entrySize <= segmentMaxSize && segment.getTotalSize() + entrySize > segmentMaxSize; } }
configEntries.add(record.getTermIndex()); totalSize += getEntrySize(entry); endIndex = entry.getIndex();
configEntries.add(record.getTermIndex()); totalSize += getEntrySize(entry); endIndex = entry.getIndex();
@Test public void testAppendEntries() throws Exception { final long start = 1000; LogSegment segment = LogSegment.newOpenSegment(null, start); long size = SegmentedRaftLog.HEADER_BYTES.length; final long max = 8 * 1024 * 1024; checkLogSegment(segment, start, start - 1, true, size, 0); // append till full long term = 0; int i = 0; List<LogEntryProto> list = new ArrayList<>(); while (size < max) { SimpleOperation op = new SimpleOperation("m" + i); LogEntryProto entry = ProtoUtils.toLogEntryProto(op.getLogEntryContent(), term, i++ + start, clientId, callId); size += getEntrySize(entry); list.add(entry); } segment.appendToOpenSegment(list.toArray(new LogEntryProto[list.size()])); Assert.assertTrue(segment.getTotalSize() >= max); checkLogSegment(segment, start, i - 1 + start, true, size, term); }
@Test public void testAppendEntries() throws Exception { final long start = 1000; LogSegment segment = LogSegment.newOpenSegment(null, start); long size = SegmentedRaftLogFormat.getHeaderLength(); final long max = 8 * 1024 * 1024; checkLogSegment(segment, start, start - 1, true, size, 0); // append till full long term = 0; int i = 0; List<LogEntryProto> list = new ArrayList<>(); while (size < max) { SimpleOperation op = new SimpleOperation("m" + i); LogEntryProto entry = ServerProtoUtils.toLogEntryProto(op.getLogEntryContent(), term, i++ + start); size += getEntrySize(entry); list.add(entry); } segment.appendToOpenSegment(list.toArray(new LogEntryProto[list.size()])); Assert.assertTrue(segment.getTotalSize() >= max); checkLogSegment(segment, start, i - 1 + start, true, size, term); }
LogEntryProto entry = ProtoUtils.toLogEntryProto(op.getLogEntryContent(), 0, 0, clientId, callId); final long entrySize = LogSegment.getEntrySize(entry);
/** * Keep appending and check if pre-allocation is correct */ @Test public void testPreallocationAndAppend() throws Exception { final SizeInBytes max = SizeInBytes.valueOf(2, TraditionalBinaryPrefix.MEGA); RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); final File file = storage.getStorageDir().getOpenLogFile(0); final byte[] content = new byte[1024]; Arrays.fill(content, (byte) 1); SimpleOperation op = new SimpleOperation(new String(content)); LogEntryProto entry = ServerProtoUtils.toLogEntryProto(op.getLogEntryContent(), 0, 0); final long entrySize = LogSegment.getEntrySize(entry); long totalSize = SegmentedRaftLogFormat.getHeaderLength(); long preallocated = 16 * 1024; try (LogOutputStream out = new LogOutputStream(file, false, max.getSize(), 16 * 1024, 10 * 1024)) { Assert.assertEquals(preallocated, file.length()); while (totalSize + entrySize < max.getSize()) { totalSize += entrySize; out.write(entry); if (totalSize > preallocated) { Assert.assertEquals("totalSize==" + totalSize, preallocated + 16 * 1024, file.length()); preallocated += 16 * 1024; } } } Assert.assertEquals(totalSize, file.length()); }
LogEntryProto entry = ProtoUtils.toLogEntryProto(op.getLogEntryContent(), 0, 0, clientId, callId); size = LogSegment.getEntrySize(entry); out.write(entry);
SimpleOperation op = new SimpleOperation(new String(content)); LogEntryProto entry = ServerProtoUtils.toLogEntryProto(op.getLogEntryContent(), 0, 0); size = LogSegment.getEntrySize(entry); out.write(entry);
private void checkLogSegment(LogSegment segment, long start, long end, boolean isOpen, long totalSize, long term) throws Exception { Assert.assertEquals(start, segment.getStartIndex()); Assert.assertEquals(end, segment.getEndIndex()); Assert.assertEquals(isOpen, segment.isOpen()); Assert.assertEquals(totalSize, segment.getTotalSize()); long offset = SegmentedRaftLog.HEADER_BYTES.length; for (long i = start; i <= end; i++) { LogSegment.LogRecord record = segment.getLogRecord(i); LogRecordWithEntry lre = segment.getEntryWithoutLoading(i); Assert.assertEquals(i, lre.getRecord().getTermIndex().getIndex()); Assert.assertEquals(term, lre.getRecord().getTermIndex().getTerm()); Assert.assertEquals(offset, record.getOffset()); LogEntryProto entry = lre.hasEntry() ? lre.getEntry() : segment.loadCache(lre.getRecord()); offset += getEntrySize(entry); } }
static void checkLogSegment(LogSegment segment, long start, long end, boolean isOpen, long totalSize, long term) throws Exception { Assert.assertEquals(start, segment.getStartIndex()); Assert.assertEquals(end, segment.getEndIndex()); Assert.assertEquals(isOpen, segment.isOpen()); Assert.assertEquals(totalSize, segment.getTotalSize()); long offset = SegmentedRaftLogFormat.getHeaderLength(); for (long i = start; i <= end; i++) { LogSegment.LogRecord record = segment.getLogRecord(i); LogRecordWithEntry lre = segment.getEntryWithoutLoading(i); Assert.assertEquals(i, lre.getRecord().getTermIndex().getIndex()); Assert.assertEquals(term, lre.getRecord().getTermIndex().getTerm()); Assert.assertEquals(offset, record.getOffset()); LogEntryProto entry = lre.hasEntry() ? lre.getEntry() : segment.loadCache(lre.getRecord()); offset += getEntrySize(entry); } }