private ServiceEndpoint createServiceEndpoint() { return new ServiceEndpoint(new ScheduledThreadPoolExecutor(2), "localhost:20001", "dc1", dependencyHealthCheck); } }
@Test public void multipleAvailZones() { ServiceEndpoint ep1 = new ServiceEndpoint(null, "1.1.1.1:80", "dc1", dependencyHealthCheck); ServiceEndpoint ep2 = new ServiceEndpoint(null, "1.1.1.2:80", "dc2", dependencyHealthCheck); ServiceEndpoint ep3 = new ServiceEndpoint(null, "1.1.1.3:80", "dc3", dependencyHealthCheck); lb.addServiceEndpoint(ep1); lb.addServiceEndpoint(ep2); lb.addServiceEndpoint(ep3); assertThat(lb.getAvailabilityZoneCount()).isEqualTo(3); }
protected HttpClientWrapper createHttpClientWrapper() { int port = locateTargetServicePort(); serviceEndpoint = new ServiceEndpoint(new ScheduledThreadPoolExecutor(1), "localhost:" + port, "", dependencyHealthCheck); HttpClientWrapper retval = new HttpClientWrapper(new ServiceProperties(), createHttpClient(), null, NoopTracerFactory.create()); retval.setLoadBalancer(this); return retval; }
@Test public void addNode() { ServiceEndpoint sep1 = new ServiceEndpoint(null, "1.1.1.1:80", "dc1", dependencyHealthCheck); ServiceEndpoint sep2 = new ServiceEndpoint(null, "1.1.1.2:80", "dc1", dependencyHealthCheck); ServiceEndpoint sep3 = new ServiceEndpoint(null, "1.1.1.3:80", "dc1", dependencyHealthCheck); list.add(sep1); list.add(sep2); list.add(sep3); assertThat(list.size()).isEqualTo(3); assertThat(list.nextAvailable()).isEqualTo(sep3); assertThat(list.nextAvailable()).isEqualTo(sep1); assertThat(list.nextAvailable()).isEqualTo(sep2); assertThat(list.nextAvailable()).isEqualTo(sep3); list.updateEndpointHealth(new ServiceEndpoint(null, "1.1.1.2:80", "dc1", dependencyHealthCheck), CircuitBreakerState.State.UNHEALTHY); list.updateEndpointHealth(new ServiceEndpoint(null, "11.11.11.12:80", "dc1", dependencyHealthCheck), CircuitBreakerState.State.UNHEALTHY); }
@Test public void verifyToString() { ServiceEndpoint sep1 = new ServiceEndpoint(null, "1.1.1.1:80", "dc1", dependencyHealthCheck); ServiceEndpointNode node = new ServiceEndpointNode(sep1); assertThat(node.toString()).isEqualTo(sep1.toString()); }
@Override public ServiceEndpoint getHealthyInstance() { return new ServiceEndpoint("localhost:" + locateTargetServicePort(), "", new CircuitBreakerState(new MockScheduledThreadPoolExecutor(0))); }
@Test public void probesOnlyGetOneRequest() { ServiceEndpoint ep1 = new ServiceEndpoint(null, "1.1.1.1:80", "dc1", dependencyHealthCheck); ep1.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_TRIPPED); ServiceEndpoint ep2 = new ServiceEndpoint(null, "1.1.1.2:80", "dc1", dependencyHealthCheck); ep2.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_PROBE); lb.addServiceEndpoint(ep1); lb.addServiceEndpoint(ep2); assertThat(lb.getHealthyInstance()).isEqualTo(ep2); assertThat(lb.getHealthyInstance()).isNull(); }
@Test public void basicHappyPath() throws Exception { //Create 2 healthy endpoints and send 4 requests. //Each endpoint should have processed 2 requests each. loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20002", "dc1", dependencyHealthCheck)); for (int i = 0; i < 4; i++) { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } assertThat(httpClient.verifyRequestsProcessed(2, "localhost:20001")).isTrue(); assertThat(httpClient.verifyRequestsProcessed(2, "localhost:20002")).isTrue(); }
@Test public void mostlyUnhealthySingleAz() { ServiceEndpoint ep1 = new ServiceEndpoint(null, "1.1.1.1:80", "dc1", dependencyHealthCheck); ep1.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_TRIPPED); ServiceEndpoint ep2 = new ServiceEndpoint(null, "1.1.1.2:80", "dc1", dependencyHealthCheck); ep2.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_TRIPPED); ServiceEndpoint ep3 = new ServiceEndpoint(null, "1.1.1.3:80", "dc1", dependencyHealthCheck); ep3.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_TRIPPED); ServiceEndpoint ep4 = new ServiceEndpoint(null, "1.1.1.4:80", "dc1", dependencyHealthCheck); ep4.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_TRIPPED); lb.addServiceEndpoint(ep1); lb.addServiceEndpoint(ep2); lb.addServiceEndpoint(ep3); lb.addServiceEndpoint(ep4); assertThat(lb.getHealthyInstance()).isNull(); ServiceEndpoint ep5 = new ServiceEndpoint(null, "1.1.1.5:80", "dc1", dependencyHealthCheck); lb.addServiceEndpoint(ep5); assertThat(lb.getHealthyInstance()).isEqualTo(ep5); }
protected ServiceEndpoint newServiceEndpoint(ConsulHealthEntry entry) { ServiceEndpoint retval = new ServiceEndpoint(executor, entry.getAddressAndPort(), entry.getAvailZone(), dependencyHealthCheck); retval.setServiceName(serviceName); if (ConsulHealthEntry.Status.Passing.equals(entry.getStatus())) { //TODO: this might need some more work. a flapping service in consul should //not bypass normal circuit breaker logic. retval.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_HEALTHY); } else { retval.setCircuitBreakerState(CircuitBreakerState.State.UNHEALTHY); } return retval; }
@Test public void multipleServicesSingleRetry() { //Create 2 servers and have them always fail. Set retries to 1. Send request. //Verify each server got issued one request. loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20002", "dc1", dependencyHealthCheck)); httpClient.makeFailing(); int failureCount = 0; try { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } catch (java.lang.Exception e) { failureCount++; } assertThat(failureCount).isEqualTo(1); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20001")).isTrue(); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20002")).isTrue(); }
@Test public void newServiceInstanceAppears() throws Exception { //Create 2 servers and send 4 requests. Each should have processed 2 requests each. //Add a new server. Send 3 more requests. Two servers should have processed 3 //and the third one just 1. loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20002", "dc1", dependencyHealthCheck)); for (int i = 0; i < 4; i++) { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20003", "dc1", dependencyHealthCheck)); for (int i = 0; i < 3; i++) { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } assertThat(httpClient.verifyRequestsProcessed(3, "localhost:20001")).isTrue(); assertThat(httpClient.verifyRequestsProcessed(3, "localhost:20002")).isTrue(); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20003")).isTrue(); }
@Test public void allFailingNoInstanceAvailable() { //Create 3 servers and have them always fail. Set retries to 4. Send request. //Each server should be tried once, then the request should fail. loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20002", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20003", "dc1", dependencyHealthCheck)); rpcClient = clientFactory.newClient(serviceName, "testing", FrameworkTest.Foobar.class). withRetries(4).build(); httpClient.makeFailing(); int failureCount = 0; try { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } catch (java.lang.Exception e) { e.printStackTrace(); failureCount++; } assertThat(failureCount).isEqualTo(1); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20001")).isTrue(); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20002")).isTrue(); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20003")).isTrue(); }
@Test public void allInstancesTimeOut() { //Create 2 servers and have both timeout. Set retries to 1. Send request. //The response from the 2nd should be returned. loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20002", "dc1", dependencyHealthCheck)); rpcClient = clientFactory.newClient(serviceName, "testing", FrameworkTest.Foobar.class). withRetries(1).build(); httpClient.makeRequestsTimeout(); int failureCount = 0; try { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } catch (RpcCallException ex) { failureCount++; assertThat(ex.getCategory()).isEqualTo(RpcCallException.Category.RequestTimedOut); } assertThat(failureCount).isEqualTo(1); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20001")).isTrue(); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20002")).isTrue(); }
@Test public void singleInstanceTimesOut() { //Create 2 servers and have one timeout. Set retries to 1. Send request. //The response from the 2nd should be returned. loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20002", "dc1", dependencyHealthCheck)); rpcClient = clientFactory.newClient(serviceName, "testing", FrameworkTest.Foobar.class). withRetries(1).build(); httpClient.makeFirstRequestTimeout(); try { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } catch (java.lang.Exception e) { e.printStackTrace(); } assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20001")).isTrue(); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20002")).isTrue(); }
@Test public void retriableError() { //Create 2 servers and have both throw an exception that is retriable. //The exception should be thrown locally loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20002", "dc1", dependencyHealthCheck)); rpcClient = clientFactory.newClient(serviceName, "testing", FrameworkTest.Foobar.class). withRetries(1).build(); httpClient.setResponseException(new RpcCallException(RpcCallException.Category. InternalServerError, "test1234").withSource("testing567")); int failureCount = 0; try { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } catch (RpcCallException ex) { failureCount++; assertThat(ex.getCategory()).isEqualTo(RpcCallException.Category.InternalServerError); assertThat(ex.getMessage()).isEqualTo("test1234"); assertThat(ex.getSource()).isEqualTo("testing567"); } assertThat(failureCount).isEqualTo(1); assertThat(httpClient.verifyRequestsProcessed(2)).isTrue(); }
@Test public void nonRetriableError() { //Create 2 servers and have the first throw an exception that is not retriable. //The exception should be thrown and not retried with the other node loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20002", "dc1", dependencyHealthCheck)); rpcClient = clientFactory.newClient(serviceName, "testing", FrameworkTest.Foobar.class). withRetries(1).build(); httpClient.setResponseException(new RpcCallException(RpcCallException.Category. InsufficientPermissions, "test").withSource("testing")); int failureCount = 0; try { rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); } catch (RpcCallException ex) { failureCount++; assertThat(ex.getCategory()).isEqualTo(RpcCallException.Category.InsufficientPermissions); assertThat(ex.getMessage()).isEqualTo("test"); } assertThat(failureCount).isEqualTo(1); assertThat(httpClient.verifyRequestsProcessed(1)).isTrue(); }
@Test public void singleServiceZeroRetries() throws Exception { //Verify retry behavior. Create 1 server and have it always fail. //Set retries to 0. Send a request, and verify server got issued one request. loadBalancer.addServiceEndpoint(new ServiceEndpoint(executor, "localhost:20001", "dc1", dependencyHealthCheck)); rpcClient = clientFactory.newClient(serviceName, "testing", FrameworkTest.Foobar.class). withRetries(0).build(); rpcClient.callSynchronous(FrameworkTest.Foobar.newBuilder().build(), new OrangeContext()); assertThat(httpClient.verifyRequestsProcessed(1, "localhost:20001")).isTrue(); }