@Override public String toString() { return name + "(c" + getCommitIndex() + ",m" + getMatchIndex() + ",n" + getNextIndex() + ", attendVote=" + attendVote + ", lastRpcSendTime=" + lastRpcSendTime.get().elapsedTimeMs() + ", lastRpcResponseTime=" + lastRpcResponseTime.get().elapsedTimeMs() + ")"; }
switch (reply.getResult()) { case SUCCESS: final long oldNextIndex = follower.getNextIndex(); final long nextIndex = reply.getNextIndex(); if (nextIndex < oldNextIndex) { follower.updateMatchIndex(nextIndex - 1); follower.updateNextIndex(nextIndex); submitEventOnSuccessAppend(); break; case INCONSISTENCY: follower.decreaseNextIndex(reply.getNextIndex()); break; case UNRECOGNIZED: LOG.warn("{} received UNRECOGNIZED AppendResult from {}", server.getId(), follower.getPeer().getId()); break;
@Override public void onNext(InstallSnapshotReplyProto reply) { LOG.debug("{} received {} response from {}", server.getId(), (!firstResponseReceived ? "the first" : "a"), follower.getPeer()); // update the last rpc time follower.updateLastRpcResponseTime(); if (!firstResponseReceived) { firstResponseReceived = true; } switch (reply.getResult()) { case SUCCESS: removePending(reply); break; case NOT_LEADER: checkResponseTerm(reply.getTerm()); break; case UNRECOGNIZED: break; } }
void updateFollowerCommitInfos(CommitInfoCache cache, List<CommitInfoProto> protos) { senders.stream().map(LogAppender::getFollower) .map(f -> cache.update(f.getPeer(), f.getCommitIndex())) .forEach(protos::add); }
/** Should the leader send appendEntries RPC to this follower? */ protected boolean shouldSendRequest() { return shouldAppendEntries(follower.getNextIndex()) || shouldHeartbeat(); }
private InstallSnapshotReplyProto installSnapshot(SnapshotInfo snapshot) throws InterruptedIOException { String requestId = UUID.randomUUID().toString(); InstallSnapshotReplyProto reply = null; try { for (InstallSnapshotRequestProto request : new SnapshotRequestIter(snapshot, requestId)) { follower.updateLastRpcSendTime(); reply = server.getServerRpc().installSnapshot(request); follower.updateLastRpcResponseTime(); if (!reply.getServerReply().getSuccess()) { return reply; } } } catch (InterruptedIOException iioe) { throw iioe; } catch (Exception ioe) { LOG.warn("{}: Failed to installSnapshot {}: {}", this, snapshot, ioe); handleException(ioe); return null; } if (reply != null) { follower.updateMatchIndex(snapshot.getTermIndex().getIndex()); follower.updateNextIndex(snapshot.getTermIndex().getIndex() + 1); LOG.info("{}: install snapshot-{} successfully on follower {}", server.getId(), snapshot.getTermIndex().getIndex(), follower.getPeer()); } return reply; }
final long leaderNextIndex = leaderLog.getNextIndex(); final long followerMatchIndex = logAppender.getFollower().getMatchIndex(); Assert.assertTrue(followerMatchIndex >= leaderNextIndex - 1); Assert.assertEquals(followerMatchIndex + 1, logAppender.getFollower().getNextIndex()); }, 10, HUNDRED_MILLIS, "assertRaftLog-" + logAppender.getFollower(), LOG)));
private void installSnapshot(SnapshotInfo snapshot) { LOG.info("{}: follower {}'s next index is {}," + " log's start index is {}, need to install snapshot", server.getId(), follower.getPeer(), follower.getNextIndex(), raftLog.getStartIndex()); if (isAppenderRunning()) { snapshotRequestObserver.onNext(request); follower.updateLastRpcSendTime(); responseHandler.addPending(request); } else { follower.setSnapshotIndex(snapshot.getTermIndex().getIndex()); LOG.info("{}: install snapshot-{} successfully on follower {}", server.getId(), snapshot.getTermIndex().getIndex(), follower.getPeer());
private InstallSnapshotReplyProto installSnapshot(SnapshotInfo snapshot) throws InterruptedIOException { String requestId = UUID.randomUUID().toString(); InstallSnapshotReplyProto reply = null; try { for (InstallSnapshotRequestProto request : new SnapshotRequestIter(snapshot, requestId)) { follower.updateLastRpcSendTime(); reply = server.getServerRpc().installSnapshot(request); follower.updateLastRpcResponseTime(); if (!reply.getServerReply().getSuccess()) { return reply; } } } catch (InterruptedIOException iioe) { throw iioe; } catch (Exception ioe) { LOG.warn("{}: Failed to installSnapshot {}: {}", this, snapshot, ioe); handleException(ioe); return null; } if (reply != null) { follower.setSnapshotIndex(snapshot.getTermIndex().getIndex()); LOG.info("{}: install snapshot-{} successfully on follower {}", server.getId(), snapshot.getTermIndex().getIndex(), follower.getPeer()); } return reply; }
/** * @return the RaftPeer (address and id) information of the followers. */ List<RaftPeer> getFollowers() { return Collections.unmodifiableList(senders.stream() .map(sender -> sender.getFollower().getPeer()) .collect(Collectors.toList())); }
/** * So far we use a simple implementation for catchup checking: * 1. If the latest rpc time of the remote peer is before 3 * max_timeout, * the peer made no progress for that long. We should fail the whole * setConfiguration request. * 2. If the peer's matching index is just behind for a small gap, and the * peer was updated recently (within max_timeout), declare the peer as * caught-up. * 3. Otherwise the peer is making progressing. Keep waiting. */ private BootStrapProgress checkProgress(FollowerInfo follower, long committed) { Preconditions.assertTrue(!follower.isAttendingVote()); final Timestamp progressTime = new Timestamp().addTimeMs(-server.getMaxTimeoutMs()); final Timestamp timeoutTime = new Timestamp().addTimeMs(-3*server.getMaxTimeoutMs()); if (follower.getLastRpcResponseTime().compareTo(timeoutTime) < 0) { LOG.debug("{} detects a follower {} timeout for bootstrapping," + " timeoutTime: {}", server.getId(), follower, timeoutTime); return BootStrapProgress.NOPROGRESS; } else if (follower.getMatchIndex() + stagingCatchupGap > committed && follower.getLastRpcResponseTime().compareTo(progressTime) > 0) { return BootStrapProgress.CAUGHTUP; } else { return BootStrapProgress.PROGRESSING; } }
LOG.info("{}: follower {}'s next index is {}," + " log's start index is {}, need to install snapshot", server.getId(), follower.getPeer(), follower.getNextIndex(), raftLog.getStartIndex()); if (isAppenderRunning() && !shouldAppendEntries(follower.getNextIndex())) { final long waitTime = getHeartbeatRemainingTime(); if (waitTime > 0) {
"reply's next index is %s, request's previous is %s", replyNextIndex, request.getPreviousLog()); updateMatchIndex = request.hasPreviousLog() && follower.getMatchIndex() < lastIndex; } else { follower.updateMatchIndex(lastIndex); submitEventOnSuccessAppend();
follower.updateLastRpcSendTime(); final AppendEntriesReplyProto r = server.getServerRpc().appendEntries(request); follower.updateLastRpcResponseTime();
protected AppendEntriesRequestProto createRequest(long callId) throws RaftLogIOException { final TermIndex previous = getPrevious(); final long heartbeatRemainingMs = getHeartbeatRemainingTime(); if (heartbeatRemainingMs <= 0L) { return leaderState.newAppendEntriesRequestProto( getFollowerId(), previous, Collections.emptyList(), !follower.isAttendingVote(), callId); } Preconditions.assertTrue(buffer.isEmpty(), () -> "buffer has " + buffer.getNumElements() + " elements."); final long leaderNext = raftLog.getNextIndex(); for (long next = follower.getNextIndex(); leaderNext > next; ) { if (!buffer.offer(raftLog.getEntryWithData(next++))) { break; } } if (buffer.isEmpty()) { return null; } final List<LogEntryProto> protos = buffer.pollList(heartbeatRemainingMs, EntryWithData::getEntry, (entry, time, exception) -> LOG.warn(this + ": Failed get " + entry + " in " + time, exception)); buffer.clear(); return leaderState.newAppendEntriesRequestProto( getFollowerId(), previous, protos, !follower.isAttendingVote(), callId); }
private long[] computeCommittedIndices(List<FollowerInfo> followers, boolean includeSelf) { final int length = includeSelf ? followers.size() + 1 : followers.size(); if (length == 0) { throw new IllegalArgumentException("followers.size() == " + followers.size() + " and includeSelf == " + includeSelf); } final long[] indices = new long[length]; for (int i = 0; i < followers.size(); i++) { indices[i] = followers.get(i).getMatchIndex(); } if (includeSelf) { // note that we also need to wait for the local disk I/O indices[length - 1] = raftLog.getLatestFlushedIndex(); } Arrays.sort(indices); return indices; }
ls.getLogAppenders().map(LogAppender::getFollower).forEach(f -> leader.addFollowerInfo(ServerProtoUtils.toServerRpcProto( f.getPeer(), f.getLastRpcResponseTime().elapsedTimeMs()))); roleInfo.setLeaderInfo(leader); });
private void clearPendingRequests(long newNextIndex) { pendingRequests.clear(); follower.decreaseNextIndex(newNextIndex); }
private void updateNextIndex(AppendEntriesRequestProto request) { final int count = request.getEntriesCount(); if (count > 0) { follower.updateNextIndex(request.getEntries(count - 1).getIndex() + 1); } }
private void onNextImpl(AppendEntriesReplyProto reply) { // update the last rpc time follower.updateLastRpcResponseTime(); if (!firstResponseReceived) { firstResponseReceived = true; } switch (reply.getResult()) { case SUCCESS: onSuccess(reply); break; case NOT_LEADER: onNotLeader(reply); break; case INCONSISTENCY: onInconsistency(reply); break; default: break; } notifyAppend(); }