private ServiceEndpoint createServiceEndpoint() { return new ServiceEndpoint(new ScheduledThreadPoolExecutor(2), "localhost:20001", "dc1", dependencyHealthCheck); } }
public void updateEndpointHealth(ServiceEndpoint ep, CircuitBreakerState.State state) { mutex.readLock().lock(); try { ServiceEndpointNode current = returnNext; int count = size; for (int i = 0; i < count; i++) { if (current.value.getHostAndPort().equals(ep.getHostAndPort())) { current.value.setCircuitBreakerState(state); return; } current = current.next; } logger.error("updateEndpointHealth: endpoint not found: {}", ep.toString()); } finally { mutex.readLock().unlock(); } }
public ServiceEndpoint nextAvailable() { mutex.writeLock().lock(); //needs write b/c it calls canServeRequests try { ServiceEndpointNode retval = returnNext; for (int i = 0; i < size; i++) { if (retval.value.canServeRequests()) { retval.value.incrementServingRequests(); returnNext = retval.next; return retval.value; } else { returnNext = returnNext.next; retval = returnNext; } } //if we got here, there are none available return null; } finally { mutex.writeLock().unlock(); } }
private HealthCheck buildFailedHealthCheck(List<ServiceEndpoint> trippedBreakers) { StringBuilder sb = new StringBuilder(); trippedBreakers.forEach(serviceEndpoint -> { sb.append(serviceEndpoint.getServiceName()).append('/') .append(serviceEndpoint.getAvailZone()).append('/') .append(serviceEndpoint.getHostAndPort()).append(' '); }); return new HealthCheck("RPC dependencies", HealthCheck.Status.WARN, sb.toString()); }
protected ServiceEndpoint newServiceEndpoint(ConsulHealthEntry entry) { ServiceEndpoint retval = new ServiceEndpoint(executor, entry.getAddressAndPort(), entry.getAvailZone(), dependencyHealthCheck); retval.setServiceName(serviceName); if (ConsulHealthEntry.Status.Passing.equals(entry.getStatus())) { //TODO: this might need some more work. a flapping service in consul should //not bypass normal circuit breaker logic. retval.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_HEALTHY); } else { retval.setCircuitBreakerState(CircuitBreakerState.State.UNHEALTHY); } return retval; }
@Test public void probesOnlyGetOneRequest() { ServiceEndpoint ep1 = new ServiceEndpoint(null, "1.1.1.1:80", "dc1", dependencyHealthCheck); ep1.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_TRIPPED); ServiceEndpoint ep2 = new ServiceEndpoint(null, "1.1.1.2:80", "dc1", dependencyHealthCheck); ep2.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_PROBE); lb.addServiceEndpoint(ep1); lb.addServiceEndpoint(ep2); assertThat(lb.getHealthyInstance()).isEqualTo(ep2); assertThat(lb.getHealthyInstance()).isNull(); }
@Test public void verifyToString() { ServiceEndpoint sep1 = new ServiceEndpoint(null, "1.1.1.1:80", "dc1", dependencyHealthCheck); ServiceEndpointNode node = new ServiceEndpointNode(sep1); assertThat(node.toString()).isEqualTo(sep1.toString()); }
@Override public String toString() { return value.toString(); } }
@Override public void updateServiceEndpoints(LoadBalancerUpdate updates) { mutex.writeLock().lock(); try { Marker logMarker = append("serviceName", this.serviceName); for (ServiceEndpoint ep : updates.getNewServices()) { logger.debug(logMarker, "Endpoint for {} became available: {}", this.serviceName, ep.getHostAndPort()); addServiceEndpoint(ep); } for (ServiceEndpoint ep : updates.getDeletedServices()) { logger.debug(logMarker, "Endpoint for {} became unavailable: {}", this.serviceName, ep.getHostAndPort()); updateEndpointHealth(ep, CircuitBreakerState.State.UNHEALTHY); } for (ServiceEndpoint ep : updates.getUpdatedServices()) { logger.debug(logMarker, "Health of endpoint {} of {} changed to {}", ep.getHostAndPort(), this.serviceName, ep.getCircuitBreakerState()); updateEndpointHealth(ep, ep.getCircuitBreakerState()); } } finally { mutex.writeLock().unlock(); } }
/** * Try to find an endpoint in our primary AZ. If none found, try further AZs. * Modifies state */ @Override public ServiceEndpoint getHealthyInstance() { if (! haveEndpoints.get()) { //wait for the first one to come in try { notificationSemaphore.tryAcquire(1, TimeUnit.SECONDS); } catch (InterruptedException e) { } } mutex.readLock().lock(); try { for (AvailabilityZone az : availabilityZones) { ServiceEndpoint next = az.nextEndpoint(); if (next != null) { logger.debug("Returning instance {} for {}", next.getHostAndPort(), serviceName); return next; } } return null; } finally { mutex.readLock().unlock(); } }
protected void updateEndpointHealth(ServiceEndpoint ep, CircuitBreakerState.State state) { for (AvailabilityZone az : availabilityZones) { if (az.getName().equals(ep.getAvailZone())) { az.updateEndpointHealth(ep, state); return; } } logger.error("updateEndpointHealth: availZone {} not found", ep.getAvailZone()); }
public void debugDump(StringBuilder sb) { mutex.writeLock().lock(); try { ServiceEndpointNode current = returnNext; for (int i = 0; i < size; i++) { sb.append(" ").append(current.toString()).append(": "). append(current.value.getCircuitBreakerState()).append("\n"); current = current.next; } } finally { mutex.writeLock().unlock(); } }
request.getServiceEndpoint().requestComplete(true); return retval; } else { request.getServiceEndpoint().requestComplete(lastStatusCode < 500);
@Test public void mostlyUnhealthySingleAz() { ServiceEndpoint ep1 = new ServiceEndpoint(null, "1.1.1.1:80", "dc1", dependencyHealthCheck); ep1.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_TRIPPED); ServiceEndpoint ep2 = new ServiceEndpoint(null, "1.1.1.2:80", "dc1", dependencyHealthCheck); ep2.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_TRIPPED); ServiceEndpoint ep3 = new ServiceEndpoint(null, "1.1.1.3:80", "dc1", dependencyHealthCheck); ep3.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_TRIPPED); ServiceEndpoint ep4 = new ServiceEndpoint(null, "1.1.1.4:80", "dc1", dependencyHealthCheck); ep4.setCircuitBreakerState(CircuitBreakerState.State.PRIMARY_TRIPPED); lb.addServiceEndpoint(ep1); lb.addServiceEndpoint(ep2); lb.addServiceEndpoint(ep3); lb.addServiceEndpoint(ep4); assertThat(lb.getHealthyInstance()).isNull(); ServiceEndpoint ep5 = new ServiceEndpoint(null, "1.1.1.5:80", "dc1", dependencyHealthCheck); lb.addServiceEndpoint(ep5); assertThat(lb.getHealthyInstance()).isEqualTo(ep5); }
public HttpRequestWrapper(String method, ServiceEndpoint instance) { this.method = method; this.instance = instance; setUri(URI.create("http://" + instance.getHostAndPort() + "/")); setHeader("X-Correlation-Id", MDC.get(OrangeContext.CORRELATION_ID)); }
public void addServiceEndpoint(ServiceEndpoint endpoint) { if (serviceEndpoints.isEmpty()) { this.name = endpoint.getAvailZone(); } serviceEndpoints.add(endpoint); }
@Override public HealthCheck getHealthCheck() { List<ServiceEndpoint> trippedBreakers = new ArrayList<>(); trippedBreakers.forEach(ep -> { if (CircuitBreakerState.isTripped(ep.getCircuitBreakerState())) { trippedBreakers.add(ep); } }); if (trippedBreakers.isEmpty()) { return new HealthCheck(); } else { return buildFailedHealthCheck(trippedBreakers); } }
protected HttpClientWrapper createHttpClientWrapper() { int port = locateTargetServicePort(); serviceEndpoint = new ServiceEndpoint(new ScheduledThreadPoolExecutor(1), "localhost:" + port, "", dependencyHealthCheck); HttpClientWrapper retval = new HttpClientWrapper(new ServiceProperties(), createHttpClient(), null, NoopTracerFactory.create()); retval.setLoadBalancer(this); return retval; }
private synchronized String getServiceBaseUrl() { if (serviceBaseUrl == null) { LoadBalancer lb = lbFactory.getLoadBalancer(serviceName); lb.waitForServiceInstance(); com.sixt.service.framework.rpc.ServiceEndpoint endpoint = lb.getHealthyInstance(); serviceBaseUrl = "http://" + endpoint.getHostAndPort(); } return serviceBaseUrl; }
/** * We rely on the object that populates us to order availability zones * with primary first, then going nearest to furthest. (implying priority) * Only to be used from this class or tests */ void addServiceEndpoint(ServiceEndpoint endpoint) { boolean found = false; for (AvailabilityZone az : availabilityZones) { if (az.getName().equals(endpoint.getAvailZone())) { az.addServiceEndpoint(endpoint); found = true; } } if (! found) { AvailabilityZone az = new AvailabilityZone(); az.addServiceEndpoint(endpoint); availabilityZones.add(az); } haveEndpoints.set(true); notificationSemaphore.release(); }