@Override public String toString() { return getStorageDir() + ""; } }
@Override public String toString() { return getStorageDir() + ""; } }
private File getSegmentFile() { return isOpen ? storage.getStorageDir().getOpenLogFile(startIndex) : storage.getStorageDir().getClosedLogFile(startIndex, endIndex); }
private File getSegmentFile() { return isOpen ? storage.getStorageDir().getOpenLogFile(startIndex) : storage.getStorageDir().getClosedLogFile(startIndex, endIndex); }
@Override public void init(RaftStorage raftStorage) throws IOException { this.raftStorage = raftStorage; this.smDir = raftStorage.getStorageDir().getStateMachineDir(); loadLatestSnapshot(); }
@Override public void init(RaftStorage raftStorage) throws IOException { this.raftStorage = raftStorage; this.smDir = raftStorage.getStorageDir().getStateMachineDir(); loadLatestSnapshot(); }
@Override void execute() throws IOException { File openFile = storage.getStorageDir().getOpenLogFile(newStartIndex); LOG.debug("{} creating new log segment {}", name, openFile); Preconditions.assertTrue(!openFile.exists(), "open file %s exists for %s", openFile, name); Preconditions.assertTrue(out == null && pendingFlushNum == 0); out = new LogOutputStream(openFile, false, segmentMaxSize, preallocatedSize, bufferSize); Preconditions.assertTrue(openFile.exists(), "Failed to create file %s for %s", openFile.getAbsolutePath(), name); }
@Override void execute() throws IOException { File openFile = storage.getStorageDir().getOpenLogFile(newStartIndex); Preconditions.assertTrue(!openFile.exists(), "open file %s exists for %s", openFile, name); Preconditions.assertTrue(out == null && pendingFlushNum == 0); out = new LogOutputStream(openFile, false, segmentMaxSize, preallocatedSize, bufferSize); Preconditions.assertTrue(openFile.exists(), "Failed to create file %s for %s", openFile.getAbsolutePath(), name); LOG.info("{}: created new log segment {}", name, openFile); }
@Override public void execute() throws IOException { IOUtils.cleanup(LOG, out); out = null; File openFile = storage.getStorageDir().getOpenLogFile(startIndex); Preconditions.assertTrue(openFile.exists(), () -> name + ": File " + openFile + " to be rolled does not exist"); if (endIndex - startIndex + 1 > 0) { // finalize the current open segment File dstFile = storage.getStorageDir().getClosedLogFile(startIndex, endIndex); Preconditions.assertTrue(!dstFile.exists()); FileUtils.move(openFile, dstFile); LOG.info("{}: Rolled log segment from {} to {}", name, openFile, dstFile); } else { // delete the file of the empty segment FileUtils.deleteFile(openFile); LOG.info("{}: Deleted empty log segment {}", name, openFile); } updateFlushedIndex(); }
static List<Path> getOpenLogFiles(RaftServerImpl server) throws Exception { return server.getState().getStorage().getStorageDir().getLogSegmentFiles().stream() .filter(LogPathAndIndex::isOpen) .map(LogPathAndIndex::getPath) .collect(Collectors.toList()); }
@Override protected void openImpl(long lastIndexInSnapshot, Consumer<LogEntryProto> consumer) throws IOException { loadLogSegments(lastIndexInSnapshot, consumer); File openSegmentFile = null; LogSegment openSegment = cache.getOpenSegment(); if (openSegment != null) { openSegmentFile = storage.getStorageDir() .getOpenLogFile(openSegment.getStartIndex()); } fileLogWorker.start(Math.max(cache.getEndIndex(), lastIndexInSnapshot), openSegmentFile); }
@Override public void open(long lastIndexInSnapshot, Consumer<LogEntryProto> consumer) throws IOException { loadLogSegments(lastIndexInSnapshot, consumer); File openSegmentFile = null; LogSegment openSegment = cache.getOpenSegment(); if (openSegment != null) { openSegmentFile = storage.getStorageDir() .getOpenLogFile(openSegment.getStartIndex()); } fileLogWorker.start(Math.max(cache.getEndIndex(), lastIndexInSnapshot), openSegmentFile); super.open(lastIndexInSnapshot, consumer); }
GroupInfoReply getGroupInfo(GroupInfoRequest request) { return new GroupInfoReply(request, getRoleInfoProto(), state.getStorage().getStorageDir().hasMetaFile(), getCommitInfos(), getGroup()); }
@Test public void testZeroSizeInProgressFile() throws Exception { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); final File file = storage.getStorageDir().getOpenLogFile(0); storage.close(); // create zero size in-progress file LOG.info("file: " + file); Assert.assertTrue(file.createNewFile()); final Path path = file.toPath(); Assert.assertTrue(Files.exists(path)); Assert.assertEquals(0, Files.size(path)); // getLogSegmentFiles should remove it. final List<RaftStorageDirectory.LogPathAndIndex> logs = storage.getStorageDir().getLogSegmentFiles(); Assert.assertEquals(0, logs.size()); Assert.assertFalse(Files.exists(path)); } }
@Test public void testZeroSizeInProgressFile() throws Exception { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); final File file = storage.getStorageDir().getOpenLogFile(0); storage.close(); // create zero size in-progress file LOG.info("file: " + file); Assert.assertTrue(file.createNewFile()); final Path path = file.toPath(); Assert.assertTrue(Files.exists(path)); Assert.assertEquals(0, Files.size(path)); // getLogSegmentFiles should remove it. final List<RaftStorageDirectory.LogPathAndIndex> logs = storage.getStorageDir().getLogSegmentFiles(); Assert.assertEquals(0, logs.size()); Assert.assertFalse(Files.exists(path)); } }
private File prepareLog(boolean isOpen, long start, int size, long term) throws IOException { RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); File file = isOpen ? storage.getStorageDir().getOpenLogFile(start) : storage.getStorageDir().getClosedLogFile(start, start + size - 1); LogEntryProto[] entries = new LogEntryProto[size]; try (LogOutputStream out = new LogOutputStream(file, false, segmentMaxSize, preallocatedSize, bufferSize)) { for (int i = 0; i < size; i++) { SimpleOperation op = new SimpleOperation("m" + i); entries[i] = ProtoUtils.toLogEntryProto(op.getLogEntryContent(), term, i + start, clientId, callId); out.write(entries[i]); } } storage.close(); return file; }
private FileChunkProto readFileChunk(FileInfo fileInfo, FileInputStream in, byte[] buf, int length, long offset, int chunkIndex) throws IOException { FileChunkProto.Builder builder = FileChunkProto.newBuilder() .setOffset(offset).setChunkIndex(chunkIndex); IOUtils.readFully(in, buf, 0, length); Path relativePath = server.getState().getStorage().getStorageDir() .relativizeToRoot(fileInfo.getPath()); builder.setFilename(relativePath.toString()); builder.setDone(offset + length == fileInfo.getFileSize()); builder.setFileDigest( ByteString.copyFrom(fileInfo.getFileDigest().getDigest())); builder.setData(ByteString.copyFrom(buf, 0, length)); return builder.build(); }
private FileChunkProto readFileChunk(FileInfo fileInfo, FileInputStream in, byte[] buf, int length, long offset, int chunkIndex) throws IOException { FileChunkProto.Builder builder = FileChunkProto.newBuilder() .setOffset(offset).setChunkIndex(chunkIndex); IOUtils.readFully(in, buf, 0, length); Path relativePath = server.getState().getStorage().getStorageDir() .relativizeToRoot(fileInfo.getPath()); builder.setFilename(relativePath.toString()); builder.setDone(offset + length == fileInfo.getFileSize()); builder.setFileDigest( ByteString.copyFrom(fileInfo.getFileDigest().getDigest())); builder.setData(ByteString.copyFrom(buf, 0, length)); return builder.build(); }
@Test public void testMetaFile() throws Exception { RaftStorage storage = new RaftStorage(storageDir, StartupOption.FORMAT); File m = storage.getStorageDir().getMetaFile(); Assert.assertTrue(m.exists()); MetaFile metaFile = new MetaFile(m); Assert.assertEquals(MetaFile.DEFAULT_TERM, metaFile.getTerm()); Assert.assertEquals(MetaFile.EMPTY_VOTEFOR, metaFile.getVotedFor()); metaFile.set(123, "peer1"); metaFile.readFile(); Assert.assertEquals(123, metaFile.getTerm()); Assert.assertEquals("peer1", metaFile.getVotedFor()); MetaFile metaFile2 = new MetaFile(m); Assert.assertFalse((Boolean) Whitebox.getInternalState(metaFile2, "loaded")); Assert.assertEquals(123, metaFile.getTerm()); Assert.assertEquals("peer1", metaFile.getVotedFor()); storage.close(); }
@Test public void testMetaFile() throws Exception { RaftStorage storage = new RaftStorage(storageDir, StartupOption.FORMAT); File m = storage.getStorageDir().getMetaFile(); Assert.assertTrue(m.exists()); MetaFile metaFile = new MetaFile(m); Assert.assertEquals(MetaFile.DEFAULT_TERM, metaFile.getTerm()); Assert.assertEquals(MetaFile.EMPTY_VOTEFOR, metaFile.getVotedFor()); metaFile.set(123, "peer1"); metaFile.readFile(); Assert.assertEquals(123, metaFile.getTerm()); Assert.assertEquals("peer1", metaFile.getVotedFor()); MetaFile metaFile2 = new MetaFile(m); Assert.assertFalse((Boolean) Whitebox.getInternalState(metaFile2, "loaded")); Assert.assertEquals(123, metaFile.getTerm()); Assert.assertEquals("peer1", metaFile.getVotedFor()); storage.close(); }