@Test public void probesOnlyGetOneRequest() { ServiceEndpoint ep1 = new ServiceEndpoint(null, "1.1.1.1:80", "dc1", dependencyHealthCheck); ep1.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_TRIPPED); ServiceEndpoint ep2 = new ServiceEndpoint(null, "1.1.1.2:80", "dc1", dependencyHealthCheck); ep2.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_PROBE); lb.addServiceEndpoint(ep1); lb.addServiceEndpoint(ep2); assertThat(lb.getHealthyInstance()).isEqualTo(ep2); assertThat(lb.getHealthyInstance()).isNull(); }
@Test public void multipleAvailZones() { ServiceEndpoint ep1 = new ServiceEndpoint(null, "1.1.1.1:80", "dc1", dependencyHealthCheck); ServiceEndpoint ep2 = new ServiceEndpoint(null, "1.1.1.2:80", "dc2", dependencyHealthCheck); ServiceEndpoint ep3 = new ServiceEndpoint(null, "1.1.1.3:80", "dc3", dependencyHealthCheck); lb.addServiceEndpoint(ep1); lb.addServiceEndpoint(ep2); lb.addServiceEndpoint(ep3); assertThat(lb.getAvailabilityZoneCount()).isEqualTo(3); }
@Override public void updateServiceEndpoints(LoadBalancerUpdate updates) { mutex.writeLock().lock(); try { Marker logMarker = append("serviceName", this.serviceName); for (ServiceEndpoint ep : updates.getNewServices()) { logger.debug(logMarker, "Endpoint for {} became available: {}", this.serviceName, ep.getHostAndPort()); addServiceEndpoint(ep); } for (ServiceEndpoint ep : updates.getDeletedServices()) { logger.debug(logMarker, "Endpoint for {} became unavailable: {}", this.serviceName, ep.getHostAndPort()); updateEndpointHealth(ep, CircuitBreakerState.State.UNHEALTHY); } for (ServiceEndpoint ep : updates.getUpdatedServices()) { logger.debug(logMarker, "Health of endpoint {} of {} changed to {}", ep.getHostAndPort(), this.serviceName, ep.getCircuitBreakerState()); updateEndpointHealth(ep, ep.getCircuitBreakerState()); } } finally { mutex.writeLock().unlock(); } }
@Test public void basicHappyPath() throws Exception { //Create 2 healthy endpoints and send 4 requests. //Each endpoint should have processed 2 requests each. loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20002", "dc1", dependencyHealthCheck)); for (int i = 0; i < 4; i++) { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } assertThat(httpClient.verifyRequestsProcessed(2, "localhost:20001")).isTrue(); assertThat(httpClient.verifyRequestsProcessed(2, "localhost:20002")).isTrue(); }
@Override public void waitForServiceInstance() { while (true) { if (getHealthyInstance() != null) { break; } try { notificationSemaphore.tryAcquire(1, 100, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { logger.warn("Thread was interrupted", e); break; } } logger.debug("Found service instance of {}", serviceName); }
@Before public void setup() { ServiceProperties properties = new ServiceProperties(); HttpClient httpClient = mock(HttpClient.class); HttpClientWrapper wrapper = new HttpClientWrapper(properties, httpClient, null, null); lb = new LoadBalancerImpl(properties, wrapper); dependencyHealthCheck = mock(ServiceDependencyHealthCheck.class); }
@Test public void multipleServicesSingleRetry() { //Create 2 servers and have them always fail. Set retries to 1. Send request. //Verify each server got issued one request. loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20002", "dc1", dependencyHealthCheck)); httpClient.makeFailing(); int failureCount = 0; try { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } catch (java.lang.Exception e) { failureCount++; } assertThat(failureCount).isEqualTo(1); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20001")).isTrue(); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20002")).isTrue(); }
@Override public ServiceEndpoint getHealthyInstanceExclude(List<ServiceEndpoint> triedEndpoints) { mutex.readLock().lock(); try { Set<ServiceEndpoint> set = new HashSet<>(triedEndpoints); Set<ServiceEndpoint> seenInstances = new HashSet<>(); while (true) { ServiceEndpoint retval = getHealthyInstance(); if (FeatureFlags.shouldDisableRpcInstanceRetry(serviceProps)) { if (seenInstances.contains(retval)) { //we've made a complete loop return null; } if (set.contains(retval)) { seenInstances.add(retval); continue; } } return retval; } } finally { mutex.readLock().unlock(); } }
@Test public void newServiceInstanceAppears() throws Exception { //Create 2 servers and send 4 requests. Each should have processed 2 requests each. //Add a new server. Send 3 more requests. Two servers should have processed 3 //and the third one just 1. loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20002", "dc1", dependencyHealthCheck)); for (int i = 0; i < 4; i++) { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20003", "dc1", dependencyHealthCheck)); for (int i = 0; i < 3; i++) { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } assertThat(httpClient.verifyRequestsProcessed(3, "localhost:20001")).isTrue(); assertThat(httpClient.verifyRequestsProcessed(3, "localhost:20002")).isTrue(); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20003")).isTrue(); }
@Test public void mostlyUnhealthySingleAz() { ServiceEndpoint ep1 = new ServiceEndpoint(null, "1.1.1.1:80", "dc1", dependencyHealthCheck); ep1.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_TRIPPED); ServiceEndpoint ep2 = new ServiceEndpoint(null, "1.1.1.2:80", "dc1", dependencyHealthCheck); ep2.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_TRIPPED); ServiceEndpoint ep3 = new ServiceEndpoint(null, "1.1.1.3:80", "dc1", dependencyHealthCheck); ep3.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_TRIPPED); ServiceEndpoint ep4 = new ServiceEndpoint(null, "1.1.1.4:80", "dc1", dependencyHealthCheck); ep4.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_TRIPPED); lb.addServiceEndpoint(ep1); lb.addServiceEndpoint(ep2); lb.addServiceEndpoint(ep3); lb.addServiceEndpoint(ep4); assertThat(lb.getHealthyInstance()).isNull(); ServiceEndpoint ep5 = new ServiceEndpoint(null, "1.1.1.5:80", "dc1", dependencyHealthCheck); lb.addServiceEndpoint(ep5); assertThat(lb.getHealthyInstance()).isEqualTo(ep5); }
@Test public void allFailingNoInstanceAvailable() { //Create 3 servers and have them always fail. Set retries to 4. Send request. //Each server should be tried once, then the request should fail. loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20002", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20003", "dc1", dependencyHealthCheck)); rpcClient = clientFactory.newClient(serviceName, "testing", FrameworkTest.Foobar.class). withRetries(4).build(); httpClient.makeFailing(); int failureCount = 0; try { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } catch (java.lang.Exception e) { e.printStackTrace(); failureCount++; } assertThat(failureCount).isEqualTo(1); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20001")).isTrue(); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20002")).isTrue(); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20003")).isTrue(); }
@Test public void allInstancesTimeOut() { //Create 2 servers and have both timeout. Set retries to 1. Send request. //The response from the 2nd should be returned. loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20002", "dc1", dependencyHealthCheck)); rpcClient = clientFactory.newClient(serviceName, "testing", FrameworkTest.Foobar.class). withRetries(1).build(); httpClient.makeRequestsTimeout(); int failureCount = 0; try { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } catch (RpcCallException ex) { failureCount++; assertThat(ex.getCategory()).isEqualTo(RpcCallException.Category.RequestTimedOut); } assertThat(failureCount).isEqualTo(1); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20001")).isTrue(); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20002")).isTrue(); }
@Test public void retriableError() { //Create 2 servers and have both throw an exception that is retriable. //The exception should be thrown locally loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20002", "dc1", dependencyHealthCheck)); rpcClient = clientFactory.newClient(serviceName, "testing", FrameworkTest.Foobar.class). withRetries(1).build(); httpClient.setResponseException(new RpcCallException(RpcCallException.Category. InternalServerError, "test1234").withSource("testing567")); int failureCount = 0; try { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } catch (RpcCallException ex) { failureCount++; assertThat(ex.getCategory()).isEqualTo(RpcCallException.Category.InternalServerError); assertThat(ex.getMessage()).isEqualTo("test1234"); assertThat(ex.getSource()).isEqualTo("testing567"); } assertThat(failureCount).isEqualTo(1); assertThat(httpClient.verifyRequestsProcessed(2)).isTrue(); }
@Test public void singleInstanceTimesOut() { //Create 2 servers and have one timeout. Set retries to 1. Send request. //The response from the 2nd should be returned. loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20002", "dc1", dependencyHealthCheck)); rpcClient = clientFactory.newClient(serviceName, "testing", FrameworkTest.Foobar.class). withRetries(1).build(); httpClient.makeFirstRequestTimeout(); try { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } catch (java.lang.Exception e) { e.printStackTrace(); } assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20001")).isTrue(); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20002")).isTrue(); }
@Test public void nonRetriableError() { //Create 2 servers and have the first throw an exception that is not retriable. //The exception should be thrown and not retried with the other node loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20002", "dc1", dependencyHealthCheck)); rpcClient = clientFactory.newClient(serviceName, "testing", FrameworkTest.Foobar.class). withRetries(1).build(); httpClient.setResponseException(new RpcCallException(RpcCallException.Category. InsufficientPermissions, "test").withSource("testing")); int failureCount = 0; try { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } catch (RpcCallException ex) { failureCount++; assertThat(ex.getCategory()).isEqualTo(RpcCallException.Category.InsufficientPermissions); assertThat(ex.getMessage()).isEqualTo("test"); } assertThat(failureCount).isEqualTo(1); assertThat(httpClient.verifyRequestsProcessed(1)).isTrue(); }
@Test public void singleServiceZeroRetries() throws Exception { //Verify retry behavior. Create 1 server and have it always fail. //Set retries to 0. Send a request, and verify server got issued one request. loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); rpcClient = clientFactory.newClient(serviceName, "testing", FrameworkTest.Foobar.class). withRetries(0).build(); rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20001")).isTrue(); }