/** * From IMessagingClient */ @Override public ListenableFuture<RapidResponse> sendMessageBestEffort(final Endpoint remote, final RapidRequest msg) { return sendMessage(remote, msg); }
/** * Adapted from https://github.com/spotify/futures-extra/.../AsyncRetrier.java */ private static <T> void handleFailure(final Supplier<ListenableFuture<T>> code, final Endpoint remote, final SettableFuture<T> future, final int retries, final Throwable t, final Runnable onCallFailure, final ExecutorService backgroundExecutor) { if (retries > 0) { startCallWithRetry(code, remote, future, retries - 1, onCallFailure, backgroundExecutor); } else { future.setException(t); } } }
/** * TODO: These timeouts should be on the Rapid side of the IMessagingClient API. * * @param msg RapidRequest * @return timeout to use for the RapidRequest message */ private int getTimeoutForMessageMs(final RapidRequest msg) { switch (msg.getContentCase()) { case PROBEMESSAGE: return settings.getGrpcProbeTimeoutMs(); case JOINMESSAGE: return settings.getGrpcJoinTimeoutMs(); default: return settings.getGrpcTimeoutMs(); } }
/** * From IMessagingClient */ @Override public ListenableFuture<RapidResponse> sendMessage(final Endpoint remote, final RapidRequest msg) { Objects.requireNonNull(remote); Objects.requireNonNull(msg); final Supplier<ListenableFuture<RapidResponse>> call = () -> { final MembershipServiceFutureStub stub = getFutureStub(remote) .withDeadlineAfter(getTimeoutForMessageMs(msg), TimeUnit.MILLISECONDS); return stub.sendRequest(msg); }; final Runnable onCallFailure = () -> channelMap.invalidate(remote); return Retries.callWithRetries(call, remote, settings.getGrpcDefaultRetries(), onCallFailure, backgroundExecutor); }
public NettyClientServer(final Endpoint listenAddress, final SharedResources resources) { this.listenAddress = listenAddress; this.outstandingRequests = CacheBuilder.newBuilder() .expireAfterAccess(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS) .build(FUTURE_LOADER); this.resources = resources; // Bootstrap a client for sending messages. If this object is being used as a server instance, // Rapid will invoke the start() method which bootstraps a server. final Bootstrap clientBootstrap = new Bootstrap(); final RemovalListener<Endpoint, ChannelFuture> removalListener = removal -> removal.getValue().channel().close() .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); this.channelCache = CacheBuilder.newBuilder() .expireAfterAccess(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS) .removalListener(removalListener) .build(new ClientChannelLoader(clientBootstrap)); final ClientHandler clientHandler = new ClientHandler(); clientBootstrap.group(resources.getEventLoopGroup()) .channel(NioSocketChannel.class) .option(ChannelOption.SO_SNDBUF, 4096) .option(ChannelOption.SO_RCVBUF, 4096) .option(ChannelOption.SO_REUSEADDR, true) .option(ChannelOption.TCP_NODELAY, true) .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) .handler(new ClientChannelInitializer(clientHandler)); }
/** * From IMessagingClient */ @Override public ListenableFuture<RapidResponse> sendMessageBestEffort(final Endpoint remote, final RapidRequest msg) { Objects.requireNonNull(msg); try { return backgroundExecutor.submit(() -> { final Supplier<ListenableFuture<RapidResponse>> call = () -> { final MembershipServiceFutureStub stub; stub = getFutureStub(remote).withDeadlineAfter(getTimeoutForMessageMs(msg), TimeUnit.MILLISECONDS); return stub.sendRequest(msg); }; final Runnable onCallFailure = () -> channelMap.invalidate(remote); return Retries.callWithRetries(call, remote, 0, onCallFailure, backgroundExecutor); }).get(); } catch (final InterruptedException | ExecutionException e) { Thread.currentThread().interrupt(); return Futures.immediateFailedFuture(e); } }
/** * Start a cluster without joining. Required to bootstrap a seed node. * * @throws IOException Thrown if we cannot successfully start a server */ public Cluster start() throws IOException { Objects.requireNonNull(listenAddress); sharedResources = new SharedResources(listenAddress); messagingServer = messagingServer != null ? messagingServer : new GrpcServer(listenAddress, sharedResources, settings.getUseInProcessTransport()); messagingClient = messagingClient != null ? messagingClient : new GrpcClient(listenAddress, sharedResources, settings); final NodeId currentIdentifier = Utils.nodeIdFromUUID(UUID.randomUUID()); final MembershipView membershipView = new MembershipView(K, Collections.singletonList(currentIdentifier), Collections.singletonList(listenAddress)); final MultiNodeCutDetector cutDetector = new MultiNodeCutDetector(K, H, L); edgeFailureDetector = edgeFailureDetector != null ? edgeFailureDetector : new PingPongFailureDetector.Factory(listenAddress, messagingClient); final Map<Endpoint, Metadata> metadataMap = metadata.getMetadataCount() > 0 ? Collections.singletonMap(listenAddress, metadata) : Collections.emptyMap(); final MembershipService membershipService = new MembershipService(listenAddress, cutDetector, membershipView, sharedResources, settings, messagingClient, edgeFailureDetector, metadataMap, subscriptions); messagingServer.setMembershipService(membershipService); messagingServer.start(); return new Cluster(messagingServer, membershipService, sharedResources, listenAddress); }
private ListenableFuture<RapidResponse> sendOnce(final Endpoint remote, final RapidRequest msg) { try { final long reqNo = counter.incrementAndGet(); final SettableFuture<RapidResponse> future = outstandingRequests.get(reqNo); final ChannelFuture f = channelCache.get(remote); ignoreFuture(f.channel().writeAndFlush(new WrappedRapidRequest(reqNo, msg), f.channel().voidPromise())); return future; } catch (final ExecutionException e) { return Futures.immediateFailedFuture(e); } }
public GrpcClient(final Endpoint address, final SharedResources sharedResources, final ISettings settings) { this.address = address; this.settings = settings; this.grpcExecutor = sharedResources.getClientChannelExecutor(); this.backgroundExecutor = sharedResources.getBackgroundExecutor(); this.eventLoopGroup = settings.getUseInProcessTransport() ? null : sharedResources.getEventLoopGroup(); final RemovalListener<Endpoint, Channel> removalListener = removal -> shutdownChannel((ManagedChannel) removal.getValue()); this.channelMap = CacheBuilder.newBuilder() .expireAfterAccess(30, TimeUnit.SECONDS) .removalListener(removalListener) .build(new CacheLoader<Endpoint, Channel>() { @Override public Channel load(final Endpoint endpoint) { return getChannel(endpoint); } }); }
/** * From IMessagingServer */ @Override public void start() { // Bootstrap a server instance final ServerBootstrap serverBootstrap = new ServerBootstrap(); final ServerHandler serverHandler = new ServerHandler(); serverBootstrap.group(resources.getEventLoopGroup(), resources.getEventLoopGroup()) .channel(NioServerSocketChannel.class) .option(ChannelOption.SO_BACKLOG, 1000) .option(ChannelOption.SO_REUSEADDR, true) .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) .childHandler(new ServerChannelInitializer(serverHandler)); try { serverChannel = serverBootstrap.bind(listenAddress.getHostname(), listenAddress.getPort()).sync(); } catch (final InterruptedException e) { Thread.currentThread().interrupt(); LOG.error("Could not start server {}", e); } }
@Override public void onSuccess(@Nullable final RapidResponse rapidResponse) { if (rapidResponse != null) { ignoreFuture(ctx.writeAndFlush(new WrappedRapidResponse(msg.count, rapidResponse), ctx.voidPromise())); } }
private MembershipServiceFutureStub getFutureStub(final Endpoint remote) { if (isShuttingDown.get()) { throw new ShuttingDownException("GrpcClient is shutting down"); } final Channel channel = channelMap.getUnchecked(remote); return MembershipServiceGrpc.newFutureStub(channel); }
/** * From IMessagingClient */ @Override public ListenableFuture<RapidResponse> sendMessage(final Endpoint remote, final RapidRequest msg) { return Futures.withTimeout(sendOnce(remote, msg), DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS, resources.getScheduledTasksExecutor()); }
/** * Defined in rapid.proto. */ @Override public void sendRequest(final RapidRequest rapidRequest, final StreamObserver<RapidResponse> responseObserver) { if (membershipService != null) { final ListenableFuture<RapidResponse> result = membershipService.handleMessage(rapidRequest); Futures.addCallback(result, new ResponseCallback(responseObserver), grpcExecutor); } else if (rapidRequest.getContentCase().equals(RapidRequest.ContentCase.PROBEMESSAGE)) { /* * This is a special case which indicates that: * 1) the system is configured to use a failure detector that relies on Rapid's probe messages * 2) the node receiving the probe message has been added to the cluster but has not yet completed * its bootstrap process (has not received its join-confirmation yet). * 3) By virtue of 2), the node is "about to be up" and therefore informs the observer that it is * still bootstrapping. This extra information may or may not be respected by the failure detector, * but is useful in large deployments. */ responseObserver.onNext(BOOTSTRAPPING_MESSAGE); responseObserver.onCompleted(); } }
private Channel getChannel(final Endpoint remote) { // TODO: allow configuring SSL/TLS Channel channel; LOG.debug("Creating channel from {} to {}", address, remote); if (settings.getUseInProcessTransport()) { channel = InProcessChannelBuilder .forName(remote.toString()) .executor(grpcExecutor) .usePlaintext(true) .idleTimeout(10, TimeUnit.SECONDS) .build(); } else { channel = NettyChannelBuilder .forAddress(remote.getHostname(), remote.getPort()) .executor(grpcExecutor) .eventLoopGroup(eventLoopGroup) .usePlaintext(true) .idleTimeout(10, TimeUnit.SECONDS) .withOption(ChannelOption.SO_REUSEADDR, true) .withOption(ChannelOption.SO_SNDBUF, DEFAULT_BUF_SIZE) .withOption(ChannelOption.SO_RCVBUF, DEFAULT_BUF_SIZE) .build(); } return channel; }
@Override public void startCluster() throws IOException, InterruptedException { final Endpoint endpoint = Endpoint.newBuilder() .setHostname(listenAddress.getHost()) .setPort(listenAddress.getPort()).build(); // To use your own messaging implementation with Rapid, supply an instance each of IMessagingClient // and IMessagingServer to Cluster.Builder.setMessagingClientServer(). // // In this example, we use an object NettyClientServer which implements both the IMessagingClient // and IMessagingServer interfaces. final NettyClientServer nettyMessaging = new NettyClientServer(endpoint); if (listenAddress.equals(seedAddress)) { cluster = new Cluster.Builder(listenAddress) .setMessagingClientAndServer(nettyMessaging, nettyMessaging) .start(); } else { cluster = new Cluster.Builder(listenAddress) .setMessagingClientAndServer(nettyMessaging, nettyMessaging) .join(seedAddress); } cluster.registerSubscription(com.vrg.rapid.ClusterEvents.VIEW_CHANGE_PROPOSAL, this::onViewChangeProposal); cluster.registerSubscription(com.vrg.rapid.ClusterEvents.VIEW_CHANGE, this::onViewChange); cluster.registerSubscription(com.vrg.rapid.ClusterEvents.KICKED, this::onKicked); }
messagingServer = messagingServer != null ? messagingServer : new GrpcServer(listenAddress, sharedResources, settings.getUseInProcessTransport()); messagingClient = messagingClient != null ? messagingClient : new GrpcClient(listenAddress, sharedResources, settings); messagingServer.start(); for (int attempt = 0; attempt < RETRIES; attempt++) {
/** * Takes a call and retries it, returning the result as soon as it completes or the exception * caught from the last retry attempt. * * Adapted from https://github.com/spotify/futures-extra/.../AsyncRetrier.java * * @param call A supplier of a ListenableFuture, representing the call being retried. * @param retries The number of retry attempts to be performed before giving up * @param <T> The type of the response. * @return Returns a ListenableFuture of type T, that hosts the result of the supplied {@code call}. */ @CanIgnoreReturnValue static <T> ListenableFuture<T> callWithRetries(final Supplier<ListenableFuture<T>> call, final Endpoint remote, final int retries, final Runnable onCallFailure, final ExecutorService backgroundExecutor) { final SettableFuture<T> settable = SettableFuture.create(); startCallWithRetry(call, remote, settable, retries, onCallFailure, backgroundExecutor); return settable; }