/** * start a stopped server again. */ public RaftServerImpl restartServer(RaftPeerId newId, boolean format) throws IOException { return restartServer(newId, group, format); }
static void killAndRestartServer(RaftPeerId id, long killSleepMs, long restartSleepMs, MiniRaftCluster cluster, Logger LOG) { try { Thread.sleep(killSleepMs); cluster.killServer(id); Thread.sleep(restartSleepMs); LOG.info("restart server: " + id); cluster.restartServer(id, false); } catch (Exception e) { ExitUtils.terminate(-1, "Failed to kill/restart server: " + id, e, LOG); } }
static CompletableFuture<Void> killAndRestartServer( RaftPeerId id, long killSleepMs, long restartSleepMs, MiniRaftCluster cluster, Logger LOG) { final CompletableFuture<Void> future = new CompletableFuture<>(); new Thread(() -> { try { Thread.sleep(killSleepMs); cluster.killServer(id); Thread.sleep(restartSleepMs); LOG.info("restart server: " + id); cluster.restartServer(id, false); future.complete(null); } catch (Exception e) { ExitUtils.terminate(-1, "Failed to kill/restart server: " + id, e, LOG); } }).start(); return future; }
static void assertCorruptedLogHeader(RaftPeerId id, File openLogFile, int partialLength, MiniRaftCluster cluster, Logger LOG) throws Exception { Preconditions.assertTrue(partialLength < SegmentedRaftLogFormat.getHeaderLength()); try(final RandomAccessFile raf = new RandomAccessFile(openLogFile, "rw")) { SegmentedRaftLogFormat.applyHeaderTo(header -> { LOG.info("header = {}", StringUtils.bytes2HexString(header)); final byte[] corrupted = new byte[header.length]; System.arraycopy(header, 0, corrupted, 0, partialLength); LOG.info("corrupted = {}", StringUtils.bytes2HexString(corrupted)); raf.write(corrupted); return null; }); } final RaftServerImpl server = cluster.restartServer(id, false); server.getProxy().close(); }
cluster.restartServer(np.getId(), false);
static void assertTruncatedLog(RaftPeerId id, File openLogFile, long lastIndex, MiniRaftCluster cluster) throws Exception { // truncate log FileUtils.truncateFile(openLogFile, openLogFile.length() - 1); final RaftServerImpl server = cluster.restartServer(id, false); // the last index should be one less than before Assert.assertEquals(lastIndex - 1, server.getState().getLog().getLastEntryTermIndex().getIndex()); server.getProxy().close(); }
continue; cluster.restartServer(s.getId(), false);
cluster.restartServer(p.getId(), null, false);
async, replication, killLeader, numMessages); for (RaftServer s : cluster.getServers()) { cluster.restartServer(s.getId(), false);
RaftServerImpl follower = followers.get(i); LOG.info(String.format("restarting follower: %s", follower.getId().toString())); cluster.restartServer(follower.getId(), false );
void runTestOldLeaderCommit(CLUSTER cluster) throws Exception { final RaftServerImpl leader = waitForLeader(cluster); final RaftPeerId leaderId = leader.getId(); final long term = leader.getState().getCurrentTerm(); List<RaftServerImpl> followers = cluster.getFollowers(); final RaftServerImpl followerToSendLog = followers.get(0); for (int i = 1; i < NUM_SERVERS - 1; i++) { RaftServerImpl follower = followers.get(i); cluster.killServer(follower.getId()); } SimpleMessage[] messages = SimpleMessage.create(1); RaftTestUtil.sendMessageInNewThread(cluster, leaderId, messages); Thread.sleep(cluster.getMaxTimeout() + 100); RaftLog followerLog = followerToSendLog.getState().getLog(); Assert.assertTrue(RaftTestUtil.logEntriesContains(followerLog, messages)); LOG.info(String.format("killing old leader: %s", leaderId.toString())); cluster.killServer(leaderId); for (int i = 1; i < 3; i++) { RaftServerImpl follower = followers.get(i); LOG.info(String.format("restarting follower: %s", follower.getId().toString())); cluster.restartServer(follower.getId(), false ); } Thread.sleep(cluster.getMaxTimeout() * 5); // confirm the server with log is elected as new leader. final RaftPeerId newLeaderId = waitForLeader(cluster).getId(); Assert.assertEquals(followerToSendLog.getId(), newLeaderId); cluster.getServerAliveStream().map(s -> s.getState().getLog()) .forEach(log -> RaftTestUtil.assertLogEntries(log, term, messages)); }
void runTestOldLeaderNotCommit(CLUSTER cluster) throws Exception { final RaftPeerId leaderId = waitForLeader(cluster).getId(); List<RaftServerImpl> followers = cluster.getFollowers(); final RaftServerImpl followerToCommit = followers.get(0); for (int i = 1; i < NUM_SERVERS - 1; i++) { RaftServerImpl follower = followers.get(i); cluster.killServer(follower.getId()); } SimpleMessage[] messages = SimpleMessage.create(1); RaftTestUtil.sendMessageInNewThread(cluster, leaderId, messages); Thread.sleep(cluster.getMaxTimeout() + 100); RaftTestUtil.logEntriesContains(followerToCommit.getState().getLog(), messages); cluster.killServer(leaderId); cluster.killServer(followerToCommit.getId()); for (int i = 1; i < NUM_SERVERS - 1; i++) { RaftServerImpl follower = followers.get(i); cluster.restartServer(follower.getId(), false ); } waitForLeader(cluster); Thread.sleep(cluster.getMaxTimeout() + 100); final Predicate<LogEntryProto> predicate = l -> l.getTerm() != 1; cluster.getServerAliveStream() .map(s -> s.getState().getLog()) .forEach(log -> RaftTestUtil.checkLogEntries(log, messages, predicate)); }
async, killLeader, numMessages); for (RaftServer s : cluster.getServers()) { cluster.restartServer(s.getId(), false);
@Test public void testOldLeaderNotCommit() throws Exception { LOG.info("Running testOldLeaderNotCommit"); final MiniRaftCluster cluster = getCluster(); final RaftPeerId leaderId = waitForLeader(cluster).getId(); List<RaftServerImpl> followers = cluster.getFollowers(); final RaftServerImpl followerToCommit = followers.get(0); for (int i = 1; i < NUM_SERVERS - 1; i++) { RaftServerImpl follower = followers.get(i); cluster.killServer(follower.getId()); } SimpleMessage[] messages = SimpleMessage.create(1); sendMessageInNewThread(cluster, messages); Thread.sleep(cluster.getMaxTimeout() + 100); logEntriesContains(followerToCommit.getState().getLog(), messages); cluster.killServer(leaderId); cluster.killServer(followerToCommit.getId()); for (int i = 1; i < NUM_SERVERS - 1; i++) { RaftServerImpl follower = followers.get(i); cluster.restartServer(follower.getId(), false ); } waitForLeader(cluster); Thread.sleep(cluster.getMaxTimeout() + 100); final Predicate<LogEntryProto> predicate = l -> l.getTerm() != 1; cluster.getServerAliveStream() .map(s -> s.getState().getLog()) .forEach(log -> RaftTestUtil.checkLogEntries(log, messages, predicate)); }
cluster.restartServer(followerId, false); cluster.restartServer(followerId, false); Assert.assertTrue(cluster.getServer(followerId).getImpl().getState().getLog() .getLastEntryTermIndex().getIndex() >= 20);
cluster.restartServer(leader, false); } else { cluster.start();
cluster.restartServer(np.getId(), false);
cluster.restartServer(followerId, false); final RaftServerImpl follower = cluster.restartServer(followerId, false); final RaftLog followerLog = follower.getState().getLog(); final long followerLastIndex = followerLog.getLastEntryTermIndex().getIndex();
cluster.restartServer(np.getId(), false);
cluster.restartServer(id, false); final RaftServerImpl server = cluster.getRaftServerImpl(id); final RaftLog raftLog = server.getState().getLog();