/** Check the storage dir and add groups*/ void initGroups(RaftGroup group) { final Optional<RaftGroup> raftGroup = Optional.ofNullable(group); final Optional<RaftGroupId> raftGroupId = raftGroup.map(RaftGroup::getGroupId); RaftServerConfigKeys.storageDirs(properties).parallelStream() .forEach((dir) -> Optional.ofNullable(dir.listFiles()) .map(Arrays::stream).orElse(Stream.empty()) .filter(File::isDirectory) .forEach(sub -> { try { LOG.info("{}: found a subdirectory {}", getId(), sub); final RaftGroupId groupId = RaftGroupId.valueOf(UUID.fromString(sub.getName())); if (!raftGroupId.filter(groupId::equals).isPresent()) { addGroup(RaftGroup.valueOf(groupId)); } } catch (Throwable t) { LOG.warn(getId() + ": Failed to initialize the group directory " + sub.getAbsolutePath() + ". Ignoring it", t); } })); raftGroup.ifPresent(this::addGroup); }
WatchRequests(Object name, RaftProperties properties) { this.name = name + "-" + getClass().getSimpleName(); final TimeDuration watchTimeout = RaftServerConfigKeys.watchTimeout(properties); this.watchTimeoutNanos = watchTimeout.to(TimeUnit.NANOSECONDS); final TimeDuration watchTimeoutDenomination = RaftServerConfigKeys.watchTimeoutDenomination(properties); this.watchTimeoutDenominationNanos = watchTimeoutDenomination.to(TimeUnit.NANOSECONDS); Preconditions.assertTrue(watchTimeoutNanos.getDuration() % watchTimeoutDenominationNanos.getDuration() == 0L, () -> "watchTimeout (=" + watchTimeout + ") is not a multiple of watchTimeoutDenomination (=" + watchTimeoutDenomination + ")."); Arrays.stream(ReplicationLevel.values()).forEach(r -> queues.put(r, new WatchQueue(r))); }
static SizeInBytes snapshotChunkSizeMax(RaftProperties properties) { return getSizeInBytes(properties::getSizeInBytes, SNAPSHOT_CHUNK_SIZE_MAX_KEY, SNAPSHOT_CHUNK_SIZE_MAX_DEFAULT, getDefaultLog()); } static void setSnapshotChunkSizeMax(RaftProperties properties, SizeInBytes maxChunkSize) {
/** * Sets the value to <code>raft.server.storage.dir</code> via * RaftServerConfigKeys and verifies it by reading directly from property. */ @Test public void testStorageDirsProperty() { final File testDir = new File( rootTestDir.get(), UUID.randomUUID().toString()); final List<File> directories = new ArrayList<>(); final RaftProperties properties = new RaftProperties(); IntStream.range(0, 10).mapToObj((i) -> new File(testDir, Integer.toString(i))).forEach(directories::add); RaftServerConfigKeys.setStorageDirs(properties, directories); final String expected = directories.stream().map(File::getAbsolutePath) .collect(Collectors.joining(",")); final String actual = properties.get(RaftServerConfigKeys.STORAGE_DIR_KEY); Assert.assertEquals(expected, actual); }
/** * Sets the value to <code>raft.server.storage.dir</code> via * RaftServerConfigKeys and also verifies the same via RaftServerConfigKeys. */ @Test public void testStorageDirs() { final File testDir = new File( rootTestDir.get(), UUID.randomUUID().toString()); final List<File> directories = new ArrayList<>(); IntStream.range(0, 10).mapToObj((i) -> new File(testDir, Integer.toString(i))).forEach(directories::add); RaftProperties properties = new RaftProperties(); RaftServerConfigKeys.setStorageDirs(properties, directories); final List<File> storageDirs = RaftServerConfigKeys.storageDirs(properties); final List<String> expectedDirs = directories.stream() .map(File::getAbsolutePath).collect(Collectors.toList()); final List<String> actualDirs = storageDirs.stream() .map(File::getAbsolutePath).collect(Collectors.toList()); actualDirs.removeAll(expectedDirs); Assert.assertEquals(directories.size(), storageDirs.size()); Assert.assertEquals(0, actualDirs.size()); } }
private RaftServerProxy newRaftServer(RaftPeerId id, RaftGroup group, boolean format) { try { final File dir = getStorageDir(id); if (format) { FileUtils.deleteFully(dir); LOG.info("Formatted directory {}", dir); } final RaftProperties prop = new RaftProperties(properties); RaftServerConfigKeys.setStorageDir(prop, dir); final StateMachine stateMachine = getStateMachine4Test(properties); return newRaftServer(id, stateMachine, group, prop); } catch (IOException e) { throw new RuntimeException(e); } }
ServerState(RaftPeerId id, RaftGroup group, RaftProperties prop, RaftServerImpl server, StateMachine stateMachine) throws IOException { this.selfId = id; this.server = server; RaftConfiguration initialConf = RaftConfiguration.newBuilder() .setConf(group.getPeers()).build(); configurationManager = new ConfigurationManager(initialConf); LOG.info("{}: {}", id, configurationManager); // use full uuid string to create a subdirectory final File dir = chooseStorageDir(RaftServerConfigKeys.storageDirs(prop), group.getGroupId().getUuid().toString()); storage = new RaftStorage(dir, RaftServerConstants.StartupOption.REGULAR); snapshotManager = new SnapshotManager(storage, id); long lastApplied = initStatemachine(stateMachine, group.getGroupId()); // On start the leader is null, start the clock now leaderId = null; this.lastNoLeaderTime = Timestamp.currentTime(); this.leaderElectionTimeoutMs = RaftServerConfigKeys.leaderElectionTimeout(prop).toIntExact(TimeUnit.MILLISECONDS); // we cannot apply log entries to the state machine in this step, since we // do not know whether the local log entries have been committed. log = initLog(id, prop, lastApplied, this::setRaftConf); RaftLog.Metadata metadata = log.loadMetadata(); currentTerm = metadata.getTerm(); votedFor = metadata.getVotedFor(); stateMachineUpdater = new StateMachineUpdater(stateMachine, server, log, lastApplied, prop); }
@Test public void testLeaderElectionDetection() throws Exception { RaftTestUtil.waitForLeader(cluster); long leaderElectionTimeout = RaftServerConfigKeys. leaderElectionTimeout(cluster.getProperties()).toIntExact(TimeUnit.MILLISECONDS); RaftServerImpl healthyFollower = cluster.getFollowers().get(1); RaftServerImpl failedFollower = cluster.getFollowers().get(0); // fail the leader and one of the followers to that quorum is not present // for next leader election to succeed. cluster.killServer(failedFollower.getId()); cluster.killServer(cluster.getLeader().getId()); // Wait to ensure that leader election is triggered and also state machine callback is triggered Thread.sleep( leaderElectionTimeout * 2); RaftProtos.RoleInfoProto roleInfoProto = SimpleStateMachine4Testing.get(healthyFollower).getLeaderElectionTimeoutInfo(); Assert.assertNotNull(roleInfoProto); Assert.assertEquals(roleInfoProto.getRole(), RaftProtos.RaftPeerRole.CANDIDATE); Assert.assertTrue(roleInfoProto.getCandidateInfo().getLastLeaderElapsedTimeMs() > leaderElectionTimeout); } }
private RaftServerProxy newRaftServer(RaftPeerId id, RaftGroup group, boolean format) { LOG.info("newRaftServer: {}, {}, format? {}", id, group, format); try { final File dir = getStorageDir(id); if (format) { FileUtils.deleteFully(dir); LOG.info("Formatted directory {}", dir); } final RaftProperties prop = new RaftProperties(properties); RaftServerConfigKeys.setStorageDirs(prop, Collections.singletonList(dir)); return newRaftServer(id, getStateMachineRegistry(properties), group, prop); } catch (IOException e) { throw new RuntimeException(e); } }
RaftServerConfigKeys.setStorageDir(prop, storageDir); RaftStorage storage = new RaftStorage(storageDir, RaftServerConstants.StartupOption.REGULAR);
@Override public void run() throws Exception { RaftPeerId peerId = RaftPeerId.valueOf(id); RaftProperties properties = new RaftProperties(); RaftPeer[] peers = getPeers(); final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort(); GrpcConfigKeys.Server.setPort(properties, port); properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE); RaftServerConfigKeys.setStorageDirs(properties, Collections.singletonList(storageDir)); StateMachine stateMachine = new ArithmeticStateMachine(); final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)), peers); RaftServer raftServer = RaftServer.newBuilder() .setServerId(RaftPeerId.valueOf(id)) .setStateMachine(stateMachine).setProperties(properties) .setGroup(raftGroup) .build(); raftServer.start(); for(; raftServer.getLifeCycleState() != LifeCycle.State.CLOSED;) { TimeUnit.SECONDS.sleep(1); } }
final int numMessages = p.numMessages; final TimeDuration watchTimeout = RaftServerConfigKeys.watchTimeout(cluster.getProperties()); final TimeDuration watchTimeoutDenomination = RaftServerConfigKeys.watchTimeoutDenomination(cluster.getProperties());
static boolean autoTriggerEnabled(RaftProperties properties) { return getBoolean(properties::getBoolean, AUTO_TRIGGER_ENABLED_KEY, AUTO_TRIGGER_ENABLED_DEFAULT, getDefaultLog()); } static void setAutoTriggerEnabled(RaftProperties properties, boolean autoTriggerThreshold) {
@Override public void run() throws Exception { RaftPeerId peerId = RaftPeerId.valueOf(id); RaftProperties properties = new RaftProperties(); RaftPeer[] peers = getPeers(); final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort(); GrpcConfigKeys.Server.setPort(properties, port); properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE); RaftServerConfigKeys.setStorageDir(properties, storageDir); StateMachine stateMachine = new ArithmeticStateMachine(); RaftGroup raftGroup = new RaftGroup(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)), peers); RaftServer raftServer = RaftServer.newBuilder() .setServerId(RaftPeerId.valueOf(id)) .setStateMachine(stateMachine).setProperties(properties) .setGroup(raftGroup) .build(); raftServer.start(); }
@Override public void run() throws Exception { RaftPeerId peerId = RaftPeerId.valueOf(id); RaftProperties properties = new RaftProperties(); RaftPeer[] peers = getPeers(); final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort(); GrpcConfigKeys.Server.setPort(properties, port); properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE); RaftServerConfigKeys.setStorageDirs(properties, Collections.singletonList(storageDir)); ConfUtils.setFile(properties::setFile, FileStoreCommon.STATEMACHINE_DIR_KEY, storageDir); StateMachine stateMachine = new FileStoreStateMachine(properties); final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)), peers); RaftServer raftServer = RaftServer.newBuilder() .setServerId(RaftPeerId.valueOf(id)) .setStateMachine(stateMachine).setProperties(properties) .setGroup(raftGroup) .build(); raftServer.start(); for(; raftServer.getLifeCycleState() != LifeCycle.State.CLOSED;) { TimeUnit.SECONDS.sleep(1); } }
static SizeInBytes preallocatedSize(RaftProperties properties) { return getSizeInBytes(properties::getSizeInBytes, PREALLOCATED_SIZE_KEY, PREALLOCATED_SIZE_DEFAULT, getDefaultLog()); } static void setPreallocatedSize(RaftProperties properties, SizeInBytes preallocatedSize) {
@Before public void setup() throws Exception { RaftProperties properties = new RaftProperties(); storageDir = getTestDir(); RaftServerConfigKeys.setStorageDir(properties, storageDir); this.segmentMaxSize = RaftServerConfigKeys.Log.segmentSizeMax(properties).getSize(); this.preallocatedSize = RaftServerConfigKeys.Log.preallocatedSize(properties).getSize(); this.bufferSize = RaftServerConfigKeys.Log.writeBufferSize(properties).getSizeInt(); }
@Before public void setup() throws Exception { RaftProperties properties = new RaftProperties(); storageDir = getTestDir(); RaftServerConfigKeys.setStorageDirs(properties, Collections.singletonList(storageDir)); this.segmentMaxSize = RaftServerConfigKeys.Log.segmentSizeMax(properties).getSize(); this.preallocatedSize = RaftServerConfigKeys.Log.preallocatedSize(properties).getSize(); this.bufferSize = RaftServerConfigKeys.Log.writeBufferSize(properties).getSizeInt(); }
static SizeInBytes writeBufferSize(RaftProperties properties) { return getSizeInBytes(properties::getSizeInBytes, WRITE_BUFFER_SIZE_KEY, WRITE_BUFFER_SIZE_DEFAULT, getDefaultLog()); } static void setWriteBufferSize(RaftProperties properties, SizeInBytes writeBufferSize) {
@Before public void setup() throws Exception { storageDir = getTestDir(); RaftProperties properties = new RaftProperties(); RaftServerConfigKeys.setStorageDir(properties, storageDir); this.segmentMaxSize = RaftServerConfigKeys.Log.segmentSizeMax(properties).getSize(); this.preallocatedSize = RaftServerConfigKeys.Log.preallocatedSize(properties).getSize(); this.bufferSize = RaftServerConfigKeys.Log.writeBufferSize(properties).getSizeInt(); }