private List<List<FollowerInfo>> divideFollowers(RaftConfiguration conf) { List<List<FollowerInfo>> lists = new ArrayList<>(2); List<FollowerInfo> listForNew = senders.stream() .filter(sender -> conf.containsInConf(sender.getFollower().getPeer().getId())) .map(LogAppender::getFollower) .collect(Collectors.toList()); lists.add(listForNew); if (conf.isTransitional()) { List<FollowerInfo> listForOld = senders.stream() .filter(sender -> conf.containsInOldConf(sender.getFollower().getPeer().getId())) .map(LogAppender::getFollower) .collect(Collectors.toList()); lists.add(listForOld); } return lists; }
/** * check if the remote peer is not included in the current conf * and should shutdown. should shutdown if all the following stands: * 1. this is a leader * 2. current conf is stable and has been committed * 3. candidate id is not included in conf * 4. candidate's last entry's index < conf's index */ private boolean shouldSendShutdown(RaftPeerId candidateId, TermIndex candidateLastEntry) { return isLeader() && getRaftConf().isStable() && getState().isConfCommitted() && !getRaftConf().containsInConf(candidateId) && candidateLastEntry.getIndex() < getRaftConf().getLogEntryIndex() && role.getLeaderState().map(ls -> !ls.isBootStrappingPeer(candidateId)).orElse(false); }
/** Build a {@link RaftConfiguration}. */ public RaftConfiguration build() { if (forceTransitional) { Preconditions.assertTrue(oldConf != null); } if (forceStable) { Preconditions.assertTrue(oldConf == null); } return new RaftConfiguration(conf, oldConf, logEntryIndex); } }
/** * @return true iff the given peer is contained in conf and, * if old conf exists, is contained in old conf. */ boolean contains(RaftPeerId peerId) { return containsInConf(peerId) && (oldConf == null || containsInOldConf(peerId)); }
/** * Update the RpcSender list based on the current configuration */ private void updateSenders(RaftConfiguration conf) { Preconditions.assertTrue(conf.isStable() && !inStagingState()); stopAndRemoveSenders(s -> !conf.containsInConf(s.getFollower().getPeer().getId())); }
private static void waitAndCheckNewConf(MiniRaftCluster cluster, RaftPeer[] peers, Collection<RaftPeerId> deadPeers) { LOG.info("waitAndCheckNewConf: peers={}, deadPeers={}, {}", Arrays.asList(peers), deadPeers, cluster.printServers()); Assert.assertNotNull(cluster.getLeader()); int numIncluded = 0; int deadIncluded = 0; final RaftConfiguration current = RaftConfiguration.newBuilder() .setConf(peers).setLogEntryIndex(0).build(); for (RaftServerImpl server : cluster.iterateServerImpls()) { LOG.info("checking {}", server); if (deadPeers != null && deadPeers.contains(server.getId())) { if (current.containsInConf(server.getId())) { deadIncluded++; } continue; } if (current.containsInConf(server.getId())) { numIncluded++; Assert.assertTrue(server.getRaftConf().isStable()); Assert.assertTrue(server.getRaftConf().hasNoChange(peers)); } else if (server.isAlive()) { // The server is successfully removed from the conf // It may not be shutdown since it may not be able to talk to the new leader (who is not in its conf). Assert.assertTrue(server.getRaftConf().isStable()); Assert.assertFalse(server.getRaftConf().containsInConf(server.getId())); } } Assert.assertEquals(peers.length, numIncluded + deadIncluded); }
RaftConfiguration generateOldNewConf(RaftConfiguration current, long logIndex) { return RaftConfiguration.newBuilder() .setConf(newConf) .setOldConf(current) .setLogEntryIndex(logIndex) .build(); }
Builder setConf(RaftConfiguration transitionalConf) { Objects.requireNonNull(transitionalConf); Preconditions.assertTrue(transitionalConf.isTransitional()); Preconditions.assertTrue(!forceTransitional); forceStable = true; return setConf(transitionalConf.conf); }
private void checkAndUpdateConfiguration(TermIndex[] entriesToCheck) { final RaftConfiguration conf = server.getRaftConf(); if (committedConf(entriesToCheck)) { if (conf.isTransitional()) { replicateNewConf(); } else { // the (new) log entry has been committed pendingRequests.replySetConfiguration(server::getCommitInfos); // if the leader is not included in the current configuration, step down if (!conf.containsInConf(server.getId())) { LOG.info("{} is not included in the new configuration {}. Step down.", server.getId(), conf); try { // leave some time for all RPC senders to send out new conf entry Thread.sleep(server.getMinTimeoutMs()); } catch (InterruptedException ignored) { } // the pending request handler will send NotLeaderException for // pending client requests when it stops server.shutdown(false); } } } }
if (!current.isStable() || leaderState.inStagingState() || !state.isConfCommitted()) { throw new ReconfigurationInProgressException( "Reconfiguration is already in progress: " + current); if (current.hasNoChange(peersInNewConf)) { pending = new PendingRequest(request); pending.setReply(new RaftClientReply(request, getCommitInfos()));
Builder setOldConf(RaftConfiguration stableConf) { Objects.requireNonNull(stableConf); Preconditions.assertTrue(stableConf.isStable()); Preconditions.assertTrue(!forceStable); forceTransitional = true; return setOldConf(stableConf.conf); }
static Collection<RaftPeer> computeNewPeers(RaftPeer[] newMembers, RaftConfiguration old) { List<RaftPeer> peers = new ArrayList<>(); for (RaftPeer p : newMembers) { if (!old.containsInConf(p.getId())) { peers.add(p); } } return peers; }
void start() { lifeCycle.transition(STARTING); state.start(); RaftConfiguration conf = getRaftConf(); if (conf != null && conf.contains(getId())) { LOG.debug("{} starts as a follower, conf={}", getId(), conf); startAsFollower(); } else { LOG.debug("{} starts with initializing state, conf={}", getId(), conf); startInitializing(); } registerMBean(getId(), getGroupId(), jmxAdapter, jmxAdapter); }
/** * Start bootstrapping new peers */ PendingRequest startSetConfiguration(SetConfigurationRequest request) { Preconditions.assertTrue(running && !inStagingState()); RaftPeer[] peersInNewConf = request.getPeersInNewConf(); Collection<RaftPeer> peersToBootStrap = RaftConfiguration .computeNewPeers(peersInNewConf, server.getRaftConf()); // add the request to the pending queue final PendingRequest pending = pendingRequests.addConfRequest(request); ConfigurationStagingState stagingState = new ConfigurationStagingState( peersToBootStrap, new PeerConfiguration(Arrays.asList(peersInNewConf))); Collection<RaftPeer> newPeers = stagingState.getNewPeers(); // set the staging state this.stagingState = stagingState; if (newPeers.isEmpty()) { applyOldNewConf(); } else { // update the LeaderState's sender list addSenders(newPeers); } return pending; }
synchronized void addConfiguration(long logIndex, RaftConfiguration conf) { final RaftConfiguration found = configurations.get(logIndex); if (found != null) { Preconditions.assertTrue(found.equals(conf)); return; } if (!configurations.isEmpty()) { final Map.Entry<Long, RaftConfiguration> lastEntry = configurations.lastEntry(); Preconditions.assertTrue(lastEntry.getKey() < logIndex, () -> "lastEntry = " + lastEntry + ", lastEntry.index >= logIndex = " + logIndex); } configurations.put(logIndex, conf); this.currentConf = conf; }
private static void waitAndCheckNewConf(MiniRaftCluster cluster, RaftPeer[] peers, Collection<String> deadPeers) throws Exception { LOG.info(cluster.printServers()); Assert.assertNotNull(cluster.getLeader()); int numIncluded = 0; int deadIncluded = 0; final RaftConfiguration current = RaftConfiguration.newBuilder() .setConf(peers).setLogEntryIndex(0).build(); for (RaftServerImpl server : cluster.iterateServerImpls()) { if (deadPeers != null && deadPeers.contains(server.getId().toString())) { if (current.containsInConf(server.getId())) { deadIncluded++; } continue; } if (current.containsInConf(server.getId())) { numIncluded++; Assert.assertTrue(server.getRaftConf().isStable()); Assert.assertTrue(server.getRaftConf().hasNoChange(peers)); } else { Assert.assertFalse(server.getId() + " is still running: " + server, server.isAlive()); } } Assert.assertEquals(peers.length, numIncluded + deadIncluded); }
RaftConfiguration generateOldNewConf(RaftConfiguration current, long logIndex) { return RaftConfiguration.newBuilder() .setConf(newConf) .setOldConf(current) .setLogEntryIndex(logIndex) .build(); }
/** * Update the RpcSender list based on the current configuration */ private void updateSenders(RaftConfiguration conf) { Preconditions.assertTrue(conf.isStable() && !inStagingState()); stopAndRemoveSenders(s -> !conf.containsInConf(s.getFollower().getPeer().getId())); }
Builder setConf(RaftConfiguration transitionalConf) { Objects.requireNonNull(transitionalConf); Preconditions.assertTrue(transitionalConf.isTransitional()); Preconditions.assertTrue(!forceTransitional); forceStable = true; return setConf(transitionalConf.conf); }
private void checkAndUpdateConfiguration(TermIndex[] entriesToCheck) { final RaftConfiguration conf = server.getRaftConf(); if (committedConf(entriesToCheck)) { if (conf.isTransitional()) { replicateNewConf(); } else { // the (new) log entry has been committed LOG.debug("{} sends success to setConfiguration request", server.getId()); pendingRequests.replySetConfiguration(); // if the leader is not included in the current configuration, step down if (!conf.containsInConf(server.getId())) { LOG.info("{} is not included in the new configuration {}. Step down.", server.getId(), conf); try { // leave some time for all RPC senders to send out new conf entry Thread.sleep(server.getMinTimeoutMs()); } catch (InterruptedException ignored) { } // the pending request handler will send NotLeaderException for // pending client requests when it stops server.shutdown(); } } } }