@Test public void testZeroSizeInProgressFile() throws Exception { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); final File file = storage.getStorageDir().getOpenLogFile(0); storage.close(); // create zero size in-progress file LOG.info("file: " + file); Assert.assertTrue(file.createNewFile()); final Path path = file.toPath(); Assert.assertTrue(Files.exists(path)); Assert.assertEquals(0, Files.size(path)); // getLogSegmentFiles should remove it. final List<RaftStorageDirectory.LogPathAndIndex> logs = storage.getStorageDir().getLogSegmentFiles(); Assert.assertEquals(0, logs.size()); Assert.assertFalse(Files.exists(path)); } }
@Test public void testNotExistent() throws IOException { FileUtils.deleteFully(storageDir); // we will format the empty directory RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); Assert.assertEquals(StorageState.NORMAL, storage.getState()); try { new RaftStorage(storageDir, StartupOption.FORMAT).close(); Assert.fail("the format should fail since the storage is still locked"); } catch (IOException e) { Assert.assertTrue(e.getMessage().contains("directory is already locked")); } storage.close(); FileUtils.deleteFully(storageDir); Assert.assertTrue(storageDir.createNewFile()); try { new RaftStorage(storageDir, StartupOption.REGULAR); Assert.fail(); } catch (IOException e) { Assert.assertTrue( e.getMessage().contains(StorageState.NON_EXISTENT.name())); } }
@Test public void testZeroSizeInProgressFile() throws Exception { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); final File file = storage.getStorageDir().getOpenLogFile(0); storage.close(); // create zero size in-progress file LOG.info("file: " + file); Assert.assertTrue(file.createNewFile()); final Path path = file.toPath(); Assert.assertTrue(Files.exists(path)); Assert.assertEquals(0, Files.size(path)); // getLogSegmentFiles should remove it. final List<RaftStorageDirectory.LogPathAndIndex> logs = storage.getStorageDir().getLogSegmentFiles(); Assert.assertEquals(0, logs.size()); Assert.assertFalse(Files.exists(path)); } }
@Test public void testNotExistent() throws IOException { FileUtils.deleteFully(storageDir); // we will format the empty directory RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); Assert.assertEquals(StorageState.NORMAL, storage.getState()); try { new RaftStorage(storageDir, StartupOption.FORMAT).close(); Assert.fail("the format should fail since the storage is still locked"); } catch (IOException e) { Assert.assertTrue(e.getMessage().contains("directory is already locked")); } storage.close(); FileUtils.deleteFully(storageDir); Assert.assertTrue(storageDir.createNewFile()); try { new RaftStorage(storageDir, StartupOption.REGULAR); Assert.fail(); } catch (IOException e) { Assert.assertTrue( e.getMessage().contains(StorageState.NON_EXISTENT.name())); } }
/** * Keep appending and check if pre-allocation is correct */ @Test public void testPreallocationAndAppend() throws Exception { final SizeInBytes max = SizeInBytes.valueOf(2, TraditionalBinaryPrefix.MEGA); RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); final File file = storage.getStorageDir().getOpenLogFile(0); final byte[] content = new byte[1024]; Arrays.fill(content, (byte) 1); SimpleOperation op = new SimpleOperation(new String(content)); LogEntryProto entry = ServerProtoUtils.toLogEntryProto(op.getLogEntryContent(), 0, 0); final long entrySize = LogSegment.getEntrySize(entry); long totalSize = SegmentedRaftLogFormat.getHeaderLength(); long preallocated = 16 * 1024; try (LogOutputStream out = new LogOutputStream(file, false, max.getSize(), 16 * 1024, 10 * 1024)) { Assert.assertEquals(preallocated, file.length()); while (totalSize + entrySize < max.getSize()) { totalSize += entrySize; out.write(entry); if (totalSize > preallocated) { Assert.assertEquals("totalSize==" + totalSize, preallocated + 16 * 1024, file.length()); preallocated += 16 * 1024; } } } Assert.assertEquals(totalSize, file.length()); }
ServerState(RaftPeerId id, RaftGroup group, RaftProperties prop, RaftServerImpl server, StateMachine stateMachine) throws IOException { this.selfId = id; this.server = server; RaftConfiguration initialConf = RaftConfiguration.newBuilder() .setConf(group.getPeers()).build(); configurationManager = new ConfigurationManager(initialConf); LOG.info("{}: {}", id, configurationManager); // use full uuid string to create a subdirectory final File dir = chooseStorageDir(RaftServerConfigKeys.storageDirs(prop), group.getGroupId().getUuid().toString()); storage = new RaftStorage(dir, RaftServerConstants.StartupOption.REGULAR); snapshotManager = new SnapshotManager(storage, id); long lastApplied = initStatemachine(stateMachine, group.getGroupId()); // On start the leader is null, start the clock now leaderId = null; this.lastNoLeaderTime = Timestamp.currentTime(); this.leaderElectionTimeoutMs = RaftServerConfigKeys.leaderElectionTimeout(prop).toIntExact(TimeUnit.MILLISECONDS); // we cannot apply log entries to the state machine in this step, since we // do not know whether the local log entries have been committed. log = initLog(id, prop, lastApplied, this::setRaftConf); RaftLog.Metadata metadata = log.loadMetadata(); currentTerm = metadata.getTerm(); votedFor = metadata.getVotedFor(); stateMachineUpdater = new StateMachineUpdater(stateMachine, server, log, lastApplied, prop); }
@Before public void setup() throws Exception { storageDir = getTestDir(); properties = new RaftProperties(); RaftServerConfigKeys.setStorageDirs(properties, Collections.singletonList(storageDir)); storage = new RaftStorage(storageDir, RaftServerConstants.StartupOption.REGULAR); this.segmentMaxSize = RaftServerConfigKeys.Log.segmentSizeMax(properties).getSize(); this.preallocatedSize = RaftServerConfigKeys.Log.preallocatedSize(properties).getSize(); this.bufferSize = RaftServerConfigKeys.Log.writeBufferSize(properties).getSizeInt(); }
ServerState(RaftPeerId id, RaftGroup group, RaftProperties prop, RaftServerImpl server, StateMachine stateMachine) throws IOException { this.selfId = id; this.server = server; RaftConfiguration initialConf = RaftConfiguration.newBuilder() .setConf(group.getPeers()).build(); configurationManager = new ConfigurationManager(initialConf); final File dir = RaftServerConfigKeys.storageDir(prop); storage = new RaftStorage(new File(dir, group.getGroupId().toString()), RaftServerConstants.StartupOption.REGULAR); snapshotManager = new SnapshotManager(storage, id); long lastApplied = initStatemachine(stateMachine, prop); leaderId = null; // we cannot apply log entries to the state machine in this step, since we // do not know whether the local log entries have been committed. log = initLog(id, prop, lastApplied, entry -> { if (entry.getLogEntryBodyCase() == CONFIGURATIONENTRY) { configurationManager.addConfiguration(entry.getIndex(), ServerProtoUtils.toRaftConfiguration(entry.getIndex(), entry.getConfigurationEntry())); } }); RaftLog.Metadata metadata = log.loadMetadata(); currentTerm = metadata.getTerm(); votedFor = metadata.getVotedFor(); stateMachineUpdater = new StateMachineUpdater(stateMachine, server, log, lastApplied, prop); }
/** * Test basic functionality: write several log entries, then read */ @Test public void testReadWriteLog() throws IOException { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); File openSegment = storage.getStorageDir().getOpenLogFile(0); long size = SegmentedRaftLog.HEADER_BYTES.length; final LogEntryProto[] entries = new LogEntryProto[100]; try (LogOutputStream out = new LogOutputStream(openSegment, false, segmentMaxSize, preallocatedSize, bufferSize)) { size += writeMessages(entries, out); } finally { storage.close(); } Assert.assertEquals(size, openSegment.length()); LogEntryProto[] readEntries = readLog(openSegment, 0, RaftServerConstants.INVALID_LOG_INDEX, true); Assert.assertArrayEquals(entries, readEntries); }
/** * Simulate the scenario that the peer is shutdown without truncating * log segment file padding. Make sure the reader can correctly handle this. */ @Test public void testReadWithPadding() throws IOException { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); File openSegment = storage.getStorageDir().getOpenLogFile(0); long size = SegmentedRaftLog.HEADER_BYTES.length; LogEntryProto[] entries = new LogEntryProto[100]; LogOutputStream out = new LogOutputStream(openSegment, false, segmentMaxSize, preallocatedSize, bufferSize); size += writeMessages(entries, out); out.flush(); // make sure the file contains padding Assert.assertEquals( RaftServerConfigKeys.Log.PREALLOCATED_SIZE_DEFAULT.getSize(), openSegment.length()); // check if the reader can correctly read the log file LogEntryProto[] readEntries = readLog(openSegment, 0, RaftServerConstants.INVALID_LOG_INDEX, true); Assert.assertArrayEquals(entries, readEntries); out.close(); Assert.assertEquals(size, openSegment.length()); }
/** * Test basic functionality: write several log entries, then read */ @Test public void testReadWriteLog() throws IOException { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); File openSegment = storage.getStorageDir().getOpenLogFile(0); long size = SegmentedRaftLogFormat.getHeaderLength(); final LogEntryProto[] entries = new LogEntryProto[100]; try (LogOutputStream out = new LogOutputStream(openSegment, false, segmentMaxSize, preallocatedSize, bufferSize)) { size += writeMessages(entries, out); } finally { storage.close(); } Assert.assertEquals(size, openSegment.length()); LogEntryProto[] readEntries = readLog(openSegment, 0, RaftServerConstants.INVALID_LOG_INDEX, true); Assert.assertArrayEquals(entries, readEntries); }
private void testLoadSegment(boolean loadInitial) throws Exception { // load an open segment File openSegmentFile = prepareLog(true, 0, 100, 0); RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); LogSegment openSegment = LogSegment.loadSegment(storage, openSegmentFile, 0, INVALID_LOG_INDEX, true, loadInitial, null); checkLogSegment(openSegment, 0, 99, true, openSegmentFile.length(), 0); storage.close(); // for open segment we currently always keep log entries in the memory Assert.assertEquals(0, openSegment.getLoadingTimes()); // load a closed segment (1000-1099) File closedSegmentFile = prepareLog(false, 1000, 100, 1); LogSegment closedSegment = LogSegment.loadSegment(storage, closedSegmentFile, 1000, 1099, false, loadInitial, null); checkLogSegment(closedSegment, 1000, 1099, false, closedSegment.getTotalSize(), 1); Assert.assertEquals(loadInitial ? 0 : 1, closedSegment.getLoadingTimes()); }
/** * Simulate the scenario that the peer is shutdown without truncating * log segment file padding. Make sure the reader can correctly handle this. */ @Test public void testReadWithPadding() throws IOException { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); File openSegment = storage.getStorageDir().getOpenLogFile(0); long size = SegmentedRaftLogFormat.getHeaderLength(); LogEntryProto[] entries = new LogEntryProto[100]; LogOutputStream out = new LogOutputStream(openSegment, false, segmentMaxSize, preallocatedSize, bufferSize); size += writeMessages(entries, out); out.flush(); // make sure the file contains padding Assert.assertEquals( RaftServerConfigKeys.Log.PREALLOCATED_SIZE_DEFAULT.getSize(), openSegment.length()); // check if the reader can correctly read the log file LogEntryProto[] readEntries = readLog(openSegment, 0, RaftServerConstants.INVALID_LOG_INDEX, true); Assert.assertArrayEquals(entries, readEntries); out.close(); Assert.assertEquals(size, openSegment.length()); }
@Before public void setup() throws Exception { storageDir = getTestDir(); properties = new RaftProperties(); RaftServerConfigKeys.setStorageDir(properties, storageDir); storage = new RaftStorage(storageDir, RaftServerConstants.StartupOption.REGULAR); this.segmentMaxSize = RaftServerConfigKeys.Log.segmentSizeMax(properties).getSize(); this.preallocatedSize = RaftServerConfigKeys.Log.preallocatedSize(properties).getSize(); this.bufferSize = RaftServerConfigKeys.Log.writeBufferSize(properties).getSizeInt(); }
private void testLoadSegment(boolean loadInitial, boolean isLastEntryPartiallyWritten) throws Exception { // load an open segment final File openSegmentFile = prepareLog(true, 0, 100, 0, isLastEntryPartiallyWritten); RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); LogSegment openSegment = LogSegment.loadSegment(storage, openSegmentFile, 0, INVALID_LOG_INDEX, true, loadInitial, null); final int delta = isLastEntryPartiallyWritten? 1: 0; checkLogSegment(openSegment, 0, 99 - delta, true, openSegmentFile.length(), 0); storage.close(); // for open segment we currently always keep log entries in the memory Assert.assertEquals(0, openSegment.getLoadingTimes()); // load a closed segment (1000-1099) final File closedSegmentFile = prepareLog(false, 1000, 100, 1, false); LogSegment closedSegment = LogSegment.loadSegment(storage, closedSegmentFile, 1000, 1099, false, loadInitial, null); checkLogSegment(closedSegment, 1000, 1099, false, closedSegment.getTotalSize(), 1); Assert.assertEquals(loadInitial ? 0 : 1, closedSegment.getLoadingTimes()); }
private File prepareLog(boolean isOpen, long start, int size, long term) throws IOException { RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); File file = isOpen ? storage.getStorageDir().getOpenLogFile(start) : storage.getStorageDir().getClosedLogFile(start, start + size - 1); LogEntryProto[] entries = new LogEntryProto[size]; try (LogOutputStream out = new LogOutputStream(file, false, segmentMaxSize, preallocatedSize, bufferSize)) { for (int i = 0; i < size; i++) { SimpleOperation op = new SimpleOperation("m" + i); entries[i] = ProtoUtils.toLogEntryProto(op.getLogEntryContent(), term, i + start, clientId, callId); out.write(entries[i]); } } storage.close(); return file; }
@Test public void testMetaFile() throws Exception { RaftStorage storage = new RaftStorage(storageDir, StartupOption.FORMAT); File m = storage.getStorageDir().getMetaFile(); Assert.assertTrue(m.exists()); MetaFile metaFile = new MetaFile(m); Assert.assertEquals(MetaFile.DEFAULT_TERM, metaFile.getTerm()); Assert.assertEquals(MetaFile.EMPTY_VOTEFOR, metaFile.getVotedFor()); metaFile.set(123, "peer1"); metaFile.readFile(); Assert.assertEquals(123, metaFile.getTerm()); Assert.assertEquals("peer1", metaFile.getVotedFor()); MetaFile metaFile2 = new MetaFile(m); Assert.assertFalse((Boolean) Whitebox.getInternalState(metaFile2, "loaded")); Assert.assertEquals(123, metaFile.getTerm()); Assert.assertEquals("peer1", metaFile.getVotedFor()); storage.close(); }
@Test public void testMetaFile() throws Exception { RaftStorage storage = new RaftStorage(storageDir, StartupOption.FORMAT); File m = storage.getStorageDir().getMetaFile(); Assert.assertTrue(m.exists()); MetaFile metaFile = new MetaFile(m); Assert.assertEquals(MetaFile.DEFAULT_TERM, metaFile.getTerm()); Assert.assertEquals(MetaFile.EMPTY_VOTEFOR, metaFile.getVotedFor()); metaFile.set(123, "peer1"); metaFile.readFile(); Assert.assertEquals(123, metaFile.getTerm()); Assert.assertEquals("peer1", metaFile.getVotedFor()); MetaFile metaFile2 = new MetaFile(m); Assert.assertFalse((Boolean) Whitebox.getInternalState(metaFile2, "loaded")); Assert.assertEquals(123, metaFile.getTerm()); Assert.assertEquals("peer1", metaFile.getVotedFor()); storage.close(); }
@Test public void testAppendLog() throws IOException { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); File openSegment = storage.getStorageDir().getOpenLogFile(0); LogEntryProto[] entries = new LogEntryProto[200]; try (LogOutputStream out = new LogOutputStream(openSegment, false, segmentMaxSize, preallocatedSize, bufferSize)) { for (int i = 0; i < 100; i++) { SimpleOperation m = new SimpleOperation("m" + i); entries[i] = ServerProtoUtils.toLogEntryProto(m.getLogEntryContent(), 0, i); out.write(entries[i]); } } try (LogOutputStream out = new LogOutputStream(openSegment, true, segmentMaxSize, preallocatedSize, bufferSize)) { for (int i = 100; i < 200; i++) { SimpleOperation m = new SimpleOperation("m" + i); entries[i] = ServerProtoUtils.toLogEntryProto(m.getLogEntryContent(), 0, i); out.write(entries[i]); } } LogEntryProto[] readEntries = readLog(openSegment, 0, RaftServerConstants.INVALID_LOG_INDEX, true); Assert.assertArrayEquals(entries, readEntries); storage.close(); }
@Test public void testAppendLog() throws IOException { final RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR); File openSegment = storage.getStorageDir().getOpenLogFile(0); LogEntryProto[] entries = new LogEntryProto[200]; try (LogOutputStream out = new LogOutputStream(openSegment, false, segmentMaxSize, preallocatedSize, bufferSize)) { for (int i = 0; i < 100; i++) { SimpleOperation m = new SimpleOperation("m" + i); entries[i] = ProtoUtils.toLogEntryProto(m.getLogEntryContent(), 0, i, clientId, callId); out.write(entries[i]); } } try (LogOutputStream out = new LogOutputStream(openSegment, true, segmentMaxSize, preallocatedSize, bufferSize)) { for (int i = 100; i < 200; i++) { SimpleOperation m = new SimpleOperation("m" + i); entries[i] = ProtoUtils.toLogEntryProto(m.getLogEntryContent(), 0, i, clientId, callId); out.write(entries[i]); } } LogEntryProto[] readEntries = readLog(openSegment, 0, RaftServerConstants.INVALID_LOG_INDEX, true); Assert.assertArrayEquals(entries, readEntries); storage.close(); }