@Override void execute() throws IOException { File openFile = storage.getStorageDir().getOpenLogFile(newStartIndex); LOG.debug("{} creating new log segment {}", name, openFile); Preconditions.assertTrue(!openFile.exists(), "open file %s exists for %s", openFile, name); Preconditions.assertTrue(out == null && pendingFlushNum == 0); out = new LogOutputStream(openFile, false, segmentMaxSize, preallocatedSize, bufferSize); Preconditions.assertTrue(openFile.exists(), "Failed to create file %s for %s", openFile.getAbsolutePath(), name); }
@Override void execute() throws IOException { File openFile = storage.getStorageDir().getOpenLogFile(newStartIndex); Preconditions.assertTrue(!openFile.exists(), "open file %s exists for %s", openFile, name); Preconditions.assertTrue(out == null && pendingFlushNum == 0); out = new LogOutputStream(openFile, false, segmentMaxSize, preallocatedSize, bufferSize); Preconditions.assertTrue(openFile.exists(), "Failed to create file %s for %s", openFile.getAbsolutePath(), name); LOG.info("{}: created new log segment {}", name, openFile); }
private File getSegmentFile() { return isOpen ? storage.getStorageDir().getOpenLogFile(startIndex) : storage.getStorageDir().getClosedLogFile(startIndex, endIndex); }
private File getSegmentFile() { return isOpen ? storage.getStorageDir().getOpenLogFile(startIndex) : storage.getStorageDir().getClosedLogFile(startIndex, endIndex); }
@Override public void execute() throws IOException { IOUtils.cleanup(LOG, out); out = null; File openFile = storage.getStorageDir().getOpenLogFile(startIndex); Preconditions.assertTrue(openFile.exists(), () -> name + ": File " + openFile + " to be rolled does not exist"); if (endIndex - startIndex + 1 > 0) { // finalize the current open segment File dstFile = storage.getStorageDir().getClosedLogFile(startIndex, endIndex); Preconditions.assertTrue(!dstFile.exists()); FileUtils.move(openFile, dstFile); LOG.info("{}: Rolled log segment from {} to {}", name, openFile, dstFile); } else { // delete the file of the empty segment FileUtils.deleteFile(openFile); LOG.info("{}: Deleted empty log segment {}", name, openFile); } updateFlushedIndex(); }
@Override protected void openImpl(long lastIndexInSnapshot, Consumer<LogEntryProto> consumer) throws IOException { loadLogSegments(lastIndexInSnapshot, consumer); File openSegmentFile = null; LogSegment openSegment = cache.getOpenSegment(); if (openSegment != null) { openSegmentFile = storage.getStorageDir() .getOpenLogFile(openSegment.getStartIndex()); } fileLogWorker.start(Math.max(cache.getEndIndex(), lastIndexInSnapshot), openSegmentFile); }
@Override public void open(long lastIndexInSnapshot, Consumer<LogEntryProto> consumer) throws IOException { loadLogSegments(lastIndexInSnapshot, consumer); File openSegmentFile = null; LogSegment openSegment = cache.getOpenSegment(); if (openSegment != null) { openSegmentFile = storage.getStorageDir() .getOpenLogFile(openSegment.getStartIndex()); } fileLogWorker.start(Math.max(cache.getEndIndex(), lastIndexInSnapshot), openSegmentFile); super.open(lastIndexInSnapshot, consumer); }
@Override public void execute() throws IOException { IOUtils.cleanup(LOG, out); out = null; Preconditions.assertTrue(segmentToClose != null); File openFile = storage.getStorageDir() .getOpenLogFile(segmentToClose.getStartIndex()); LOG.debug("{} finalizing log segment {}", name, openFile); Preconditions.assertTrue(openFile.exists(), () -> name + ": File " + openFile + " does not exist, segmentToClose=" + segmentToClose.toDebugString()); if (segmentToClose.numOfEntries() > 0) { // finalize the current open segment File dstFile = storage.getStorageDir().getClosedLogFile( segmentToClose.getStartIndex(), segmentToClose.getEndIndex()); Preconditions.assertTrue(!dstFile.exists()); FileUtils.move(openFile, dstFile); } else { // delete the file of the empty segment FileUtils.deleteFile(openFile); } updateFlushedIndex(); }
@Test public void testZeroSizeInProgressFile() throws Exception { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); final File file = storage.getStorageDir().getOpenLogFile(0); storage.close(); // create zero size in-progress file LOG.info("file: " + file); Assert.assertTrue(file.createNewFile()); final Path path = file.toPath(); Assert.assertTrue(Files.exists(path)); Assert.assertEquals(0, Files.size(path)); // getLogSegmentFiles should remove it. final List<RaftStorageDirectory.LogPathAndIndex> logs = storage.getStorageDir().getLogSegmentFiles(); Assert.assertEquals(0, logs.size()); Assert.assertFalse(Files.exists(path)); } }
@Test public void testZeroSizeInProgressFile() throws Exception { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); final File file = storage.getStorageDir().getOpenLogFile(0); storage.close(); // create zero size in-progress file LOG.info("file: " + file); Assert.assertTrue(file.createNewFile()); final Path path = file.toPath(); Assert.assertTrue(Files.exists(path)); Assert.assertEquals(0, Files.size(path)); // getLogSegmentFiles should remove it. final List<RaftStorageDirectory.LogPathAndIndex> logs = storage.getStorageDir().getLogSegmentFiles(); Assert.assertEquals(0, logs.size()); Assert.assertFalse(Files.exists(path)); } }
private LogEntryProto[] prepareLog(List<SegmentRange> list) throws IOException { List<LogEntryProto> entryList = new ArrayList<>(); for (SegmentRange range : list) { File file = range.isOpen ? storage.getStorageDir().getOpenLogFile(range.start) : storage.getStorageDir().getClosedLogFile(range.start, range.end); final int size = (int) (range.end - range.start + 1); LogEntryProto[] entries = new LogEntryProto[size]; try (LogOutputStream out = new LogOutputStream(file, false, segmentMaxSize, preallocatedSize, bufferSize)) { for (int i = 0; i < size; i++) { SimpleOperation m = new SimpleOperation("m" + (i + range.start)); entries[i] = ServerProtoUtils.toLogEntryProto(m.getLogEntryContent(), range.term, i + range.start); out.write(entries[i]); } } Collections.addAll(entryList, entries); } return entryList.toArray(new LogEntryProto[entryList.size()]); }
private LogEntryProto[] prepareLog(List<SegmentRange> list) throws IOException { List<LogEntryProto> entryList = new ArrayList<>(); for (SegmentRange range : list) { File file = range.isOpen ? storage.getStorageDir().getOpenLogFile(range.start) : storage.getStorageDir().getClosedLogFile(range.start, range.end); final int size = (int) (range.end - range.start + 1); LogEntryProto[] entries = new LogEntryProto[size]; try (LogOutputStream out = new LogOutputStream(file, false, segmentMaxSize, preallocatedSize, bufferSize)) { for (int i = 0; i < size; i++) { SimpleOperation m = new SimpleOperation("m" + (i + range.start)); entries[i] = ProtoUtils.toLogEntryProto(m.getLogEntryContent(), range.term, i + range.start, clientId, callId); out.write(entries[i]); } } Collections.addAll(entryList, entries); } return entryList.toArray(new LogEntryProto[entryList.size()]); }
final SizeInBytes max = SizeInBytes.valueOf(2, TraditionalBinaryPrefix.MEGA); RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); final File file = storage.getStorageDir().getOpenLogFile(0);
/** * Keep appending and check if pre-allocation is correct */ @Test public void testPreallocationAndAppend() throws Exception { final SizeInBytes max = SizeInBytes.valueOf(2, TraditionalBinaryPrefix.MEGA); RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); final File file = storage.getStorageDir().getOpenLogFile(0); final byte[] content = new byte[1024]; Arrays.fill(content, (byte) 1); SimpleOperation op = new SimpleOperation(new String(content)); LogEntryProto entry = ServerProtoUtils.toLogEntryProto(op.getLogEntryContent(), 0, 0); final long entrySize = LogSegment.getEntrySize(entry); long totalSize = SegmentedRaftLogFormat.getHeaderLength(); long preallocated = 16 * 1024; try (LogOutputStream out = new LogOutputStream(file, false, max.getSize(), 16 * 1024, 10 * 1024)) { Assert.assertEquals(preallocated, file.length()); while (totalSize + entrySize < max.getSize()) { totalSize += entrySize; out.write(entry); if (totalSize > preallocated) { Assert.assertEquals("totalSize==" + totalSize, preallocated + 16 * 1024, file.length()); preallocated += 16 * 1024; } } } Assert.assertEquals(totalSize, file.length()); }
/** * Test basic functionality: write several log entries, then read */ @Test public void testReadWriteLog() throws IOException { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); File openSegment = storage.getStorageDir().getOpenLogFile(0); long size = SegmentedRaftLog.HEADER_BYTES.length; final LogEntryProto[] entries = new LogEntryProto[100]; try (LogOutputStream out = new LogOutputStream(openSegment, false, segmentMaxSize, preallocatedSize, bufferSize)) { size += writeMessages(entries, out); } finally { storage.close(); } Assert.assertEquals(size, openSegment.length()); LogEntryProto[] readEntries = readLog(openSegment, 0, RaftServerConstants.INVALID_LOG_INDEX, true); Assert.assertArrayEquals(entries, readEntries); }
/** * Simulate the scenario that the peer is shutdown without truncating * log segment file padding. Make sure the reader can correctly handle this. */ @Test public void testReadWithPadding() throws IOException { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); File openSegment = storage.getStorageDir().getOpenLogFile(0); long size = SegmentedRaftLog.HEADER_BYTES.length; LogEntryProto[] entries = new LogEntryProto[100]; LogOutputStream out = new LogOutputStream(openSegment, false, segmentMaxSize, preallocatedSize, bufferSize); size += writeMessages(entries, out); out.flush(); // make sure the file contains padding Assert.assertEquals( RaftServerConfigKeys.Log.PREALLOCATED_SIZE_DEFAULT.getSize(), openSegment.length()); // check if the reader can correctly read the log file LogEntryProto[] readEntries = readLog(openSegment, 0, RaftServerConstants.INVALID_LOG_INDEX, true); Assert.assertArrayEquals(entries, readEntries); out.close(); Assert.assertEquals(size, openSegment.length()); }
/** * Test basic functionality: write several log entries, then read */ @Test public void testReadWriteLog() throws IOException { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); File openSegment = storage.getStorageDir().getOpenLogFile(0); long size = SegmentedRaftLogFormat.getHeaderLength(); final LogEntryProto[] entries = new LogEntryProto[100]; try (LogOutputStream out = new LogOutputStream(openSegment, false, segmentMaxSize, preallocatedSize, bufferSize)) { size += writeMessages(entries, out); } finally { storage.close(); } Assert.assertEquals(size, openSegment.length()); LogEntryProto[] readEntries = readLog(openSegment, 0, RaftServerConstants.INVALID_LOG_INDEX, true); Assert.assertArrayEquals(entries, readEntries); }
/** * Simulate the scenario that the peer is shutdown without truncating * log segment file padding. Make sure the reader can correctly handle this. */ @Test public void testReadWithPadding() throws IOException { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); File openSegment = storage.getStorageDir().getOpenLogFile(0); long size = SegmentedRaftLogFormat.getHeaderLength(); LogEntryProto[] entries = new LogEntryProto[100]; LogOutputStream out = new LogOutputStream(openSegment, false, segmentMaxSize, preallocatedSize, bufferSize); size += writeMessages(entries, out); out.flush(); // make sure the file contains padding Assert.assertEquals( RaftServerConfigKeys.Log.PREALLOCATED_SIZE_DEFAULT.getSize(), openSegment.length()); // check if the reader can correctly read the log file LogEntryProto[] readEntries = readLog(openSegment, 0, RaftServerConstants.INVALID_LOG_INDEX, true); Assert.assertArrayEquals(entries, readEntries); out.close(); Assert.assertEquals(size, openSegment.length()); }
private File prepareLog(boolean isOpen, long start, int size, long term) throws IOException { RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); File file = isOpen ? storage.getStorageDir().getOpenLogFile(start) : storage.getStorageDir().getClosedLogFile(start, start + size - 1); LogEntryProto[] entries = new LogEntryProto[size]; try (LogOutputStream out = new LogOutputStream(file, false, segmentMaxSize, preallocatedSize, bufferSize)) { for (int i = 0; i < size; i++) { SimpleOperation op = new SimpleOperation("m" + i); entries[i] = ProtoUtils.toLogEntryProto(op.getLogEntryContent(), term, i + start, clientId, callId); out.write(entries[i]); } } storage.close(); return file; }
@Test public void testAppendLog() throws IOException { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); File openSegment = storage.getStorageDir().getOpenLogFile(0); LogEntryProto[] entries = new LogEntryProto[200]; try (LogOutputStream out = new LogOutputStream(openSegment, false, segmentMaxSize, preallocatedSize, bufferSize)) { for (int i = 0; i < 100; i++) { SimpleOperation m = new SimpleOperation("m" + i); entries[i] = ServerProtoUtils.toLogEntryProto(m.getLogEntryContent(), 0, i); out.write(entries[i]); } } try (LogOutputStream out = new LogOutputStream(openSegment, true, segmentMaxSize, preallocatedSize, bufferSize)) { for (int i = 100; i < 200; i++) { SimpleOperation m = new SimpleOperation("m" + i); entries[i] = ServerProtoUtils.toLogEntryProto(m.getLogEntryContent(), 0, i); out.write(entries[i]); } } LogEntryProto[] readEntries = readLog(openSegment, 0, RaftServerConstants.INVALID_LOG_INDEX, true); Assert.assertArrayEquals(entries, readEntries); storage.close(); }