public ServiceEndpoint(ScheduledThreadPoolExecutor executor, String hostAndPort, String availZone, ServiceDependencyHealthCheck dependencyHealthCheck) { this(hostAndPort, availZone, new CircuitBreakerState(executor)); dependencyHealthCheck.monitorServiceEndpoint(this); }
public boolean canServeRequests() { return circuitBreaker.canServeRequests(servingRequests.get() > 0); }
public CircuitBreakerState.State getCircuitBreakerState() { return circuitBreaker.getState(); }
@Test public void primaryProbeToHealthy() { breaker.requestComplete(false); breaker.requestComplete(false); breaker.requestComplete(false); assertThat(breaker.getState()).isEqualTo(PRIMARY_TRIPPED); executor.tick(PRIMARY_TRIP_TIME); assertThat(breaker.getState()).isEqualTo(PRIMARY_PROBE); breaker.requestComplete(true); assertThat(breaker.getState()).isEqualTo(SECONDARY_HEALTHY); executor.tick(PRIMARY_TRIP_TIME); assertThat(breaker.getState()).isEqualTo(PRIMARY_HEALTHY); }
@Test public void intermittentFailuresDontTrip() { for (int i = 0; i < 10; i++) { assertThat(breaker.canServeRequests(false)).isTrue(); breaker.requestComplete(true); assertThat(breaker.canServeRequests(false)).isTrue(); breaker.requestComplete(false); } assertThat(breaker.canServeRequests(false)).isTrue(); }
try { synchronized (CircuitBreakerState.this) { if (CircuitBreakerState.this.getState().equals(fromState)) { logger.debug("CircuitBreaker timer elapsed -> {}", toState); CircuitBreakerState.this.setState(toState); if (toState.equals(SECONDARY_HEALTHY)) { executor.schedule(secondaryHealthyToPrimaryHealthy, isTripped(CircuitBreakerState.this.getState())) { } else if (CircuitBreakerState.this.getState().equals(PRIMARY_HEALTHY) && fromState.equals(PRIMARY_TRIPPED) && toState.equals(PRIMARY_PROBE)) { } else if (CircuitBreakerState.this.getState().equals(SECONDARY_HEALTHY) && fromState.equals(SECONDARY_TRIPPED) && toState.equals(SECONDARY_PROBE)) { } else if (CircuitBreakerState.this.getState().equals(TERTIARY_HEALTHY) && fromState.equals(TERTIARY_TRIPPED) && toState.equals(TERTIARY_PROBE)) { fromState, toState, getState());
@Test public void testCanServeRequests() { assertThat(breaker.canServeRequests(false)).isTrue(); breaker.setState(CircuitBreakerState.State.SECONDARY_HEALTHY); assertThat(breaker.canServeRequests(false)).isTrue(); breaker.setState(CircuitBreakerState.State.TERTIARY_HEALTHY); assertThat(breaker.canServeRequests(false)).isTrue(); breaker.setState(CircuitBreakerState.State.PRIMARY_TRIPPED); assertThat(breaker.canServeRequests(false)).isFalse(); breaker.setState(CircuitBreakerState.State.SECONDARY_TRIPPED); assertThat(breaker.canServeRequests(false)).isFalse(); breaker.setState(CircuitBreakerState.State.TERTIARY_TRIPPED); assertThat(breaker.canServeRequests(false)).isFalse(); breaker.setState(CircuitBreakerState.State.UNHEALTHY); assertThat(breaker.canServeRequests(false)).isFalse(); breaker.setState(CircuitBreakerState.State.PRIMARY_PROBE); assertThat(breaker.canServeRequests(false)).isTrue(); breaker.setState(CircuitBreakerState.State.SECONDARY_PROBE); assertThat(breaker.canServeRequests(false)).isTrue(); breaker.setState(CircuitBreakerState.State.TERTIARY_PROBE); assertThat(breaker.canServeRequests(false)).isTrue(); breaker.setState(CircuitBreakerState.State.PRIMARY_PROBE); assertThat(breaker.canServeRequests(true)).isFalse(); breaker.setState(CircuitBreakerState.State.SECONDARY_PROBE); assertThat(breaker.canServeRequests(true)).isFalse(); breaker.setState(CircuitBreakerState.State.TERTIARY_PROBE); assertThat(breaker.canServeRequests(true)).isFalse(); }
if (hasFailed()) { logger.debug("CircuitBreaker tripped!"); if (state.equals(PRIMARY_HEALTHY)) { setState(PRIMARY_TRIPPED); executor.schedule(primaryTrippedToPrimaryProbe, PRIMARY_TRIP_TIME, TimeUnit.SECONDS); } else if (state.equals(SECONDARY_HEALTHY)) { setState(SECONDARY_TRIPPED); executor.schedule(secondaryTrippedToSecondaryProbe, SECONDARY_TRIP_TIME, TimeUnit.SECONDS); } else { setState(TERTIARY_TRIPPED); executor.schedule(tertiaryTrippedToTertiaryProbe, TERTIARY_TRIP_TIME, TimeUnit.SECONDS); if (resultGood) { logger.debug("CircuitBreaker was tripped, became healthy"); setState(SECONDARY_HEALTHY); responseHistory.clear(); executor.schedule(secondaryHealthyToPrimaryHealthy, } else { logger.debug("CircuitBreaker tripped!"); setState(SECONDARY_TRIPPED); executor.schedule(secondaryTrippedToSecondaryProbe, SECONDARY_TRIP_TIME, TimeUnit.SECONDS); if (resultGood) { logger.debug("CircuitBreaker was tripped, became healthy");
public void setCircuitBreakerState(CircuitBreakerState.State state) { this.circuitBreaker.setState(state); }
public void requestComplete(boolean success) { servingRequests.decrementAndGet(); circuitBreaker.requestComplete(success); }
@Override public HealthCheck getHealthCheck() { List<ServiceEndpoint> trippedBreakers = new ArrayList<>(); trippedBreakers.forEach(ep -> { if (CircuitBreakerState.isTripped(ep.getCircuitBreakerState())) { trippedBreakers.add(ep); } }); if (trippedBreakers.isEmpty()) { return new HealthCheck(); } else { return buildFailedHealthCheck(trippedBreakers); } }
@Test public void healthyToTripped() { breaker.requestComplete(false); breaker.requestComplete(false); breaker.requestComplete(false); assertThat(breaker.getState()).isEqualTo(PRIMARY_TRIPPED); executor.tick(PRIMARY_TRIP_TIME); assertThat(breaker.getState()).isEqualTo(PRIMARY_PROBE); breaker.requestComplete(false); assertThat(breaker.getState()).isEqualTo(SECONDARY_TRIPPED); executor.tick(SECONDARY_TRIP_TIME); assertThat(breaker.getState()).isEqualTo(SECONDARY_PROBE); breaker.requestComplete(true); assertThat(breaker.getState()).isEqualTo(SECONDARY_HEALTHY); breaker.requestComplete(false); breaker.requestComplete(false); breaker.requestComplete(false); assertThat(breaker.getState()).isEqualTo(SECONDARY_TRIPPED); executor.tick(SECONDARY_TRIP_TIME); assertThat(breaker.getState()).isEqualTo(SECONDARY_PROBE); breaker.requestComplete(false); assertThat(breaker.getState()).isEqualTo(TERTIARY_TRIPPED); executor.tick(TERTIARY_TRIP_TIME); assertThat(breaker.getState()).isEqualTo(TERTIARY_PROBE); breaker.requestComplete(true); assertThat(breaker.getState()).isEqualTo(TERTIARY_HEALTHY); breaker.requestComplete(false); breaker.requestComplete(false); breaker.requestComplete(false); assertThat(breaker.getState()).isEqualTo(TERTIARY_TRIPPED);
@Test public void testHasFailed() { for (int i = 0; i < CircuitBreakerState.HISTORY_SIZE - 1; i++) { breaker.requestComplete(false); assertThat(breaker.canServeRequests(false)).isTrue(); } breaker.requestComplete(false); assertThat(breaker.canServeRequests(false)).isFalse(); }
@Test public void stateChangerRespectsFailures() { breaker.requestComplete(false); breaker.requestComplete(false); breaker.requestComplete(false); assertThat(breaker.getState()).isEqualTo(PRIMARY_TRIPPED); executor.tick(PRIMARY_TRIP_TIME); assertThat(breaker.getState()).isEqualTo(PRIMARY_PROBE); breaker.requestComplete(false); assertThat(breaker.getState()).isEqualTo(SECONDARY_TRIPPED); executor.tick(SECONDARY_TRIP_TIME); assertThat(breaker.getState()).isEqualTo(SECONDARY_PROBE); breaker.requestComplete(true); assertThat(breaker.getState()).isEqualTo(SECONDARY_HEALTHY); breaker.requestComplete(false); breaker.requestComplete(false); breaker.requestComplete(false); executor.tick(PRIMARY_TRIP_TIME); assertThat(executor.scheduledTasks).hasSize(2); executor.tick(PRIMARY_TRIP_TIME); assertThat(breaker.getState()).isEqualTo(SECONDARY_PROBE); breaker.requestComplete(false); assertThat(breaker.getState()).isEqualTo(TERTIARY_TRIPPED); executor.tick(TERTIARY_TRIP_TIME); assertThat(breaker.getState()).isEqualTo(TERTIARY_PROBE); breaker.requestComplete(true); assertThat(breaker.getState()).isEqualTo(TERTIARY_HEALTHY); breaker.requestComplete(false); breaker.requestComplete(false); breaker.requestComplete(false);
@Override public ServiceEndpoint getHealthyInstance() { return new ServiceEndpoint("localhost:" + locateTargetServicePort(), "", new CircuitBreakerState(new MockScheduledThreadPoolExecutor(0))); }
@Test public void constructedHealthy() { assertThat(breaker.getState()).isEqualTo(CircuitBreakerState.State.PRIMARY_HEALTHY); }
@Test public void toHellAndBack() { for (int i = 0; i < 3; i++) { assertThat(breaker.getState()).isEqualTo(PRIMARY_HEALTHY); breaker.requestComplete(false); breaker.requestComplete(false); assertThat(breaker.getState()).isEqualTo(PRIMARY_HEALTHY); breaker.requestComplete(false); assertThat(breaker.getState()).isEqualTo(PRIMARY_TRIPPED); executor.tick(PRIMARY_TRIP_TIME - 1); assertThat(breaker.getState()).isEqualTo(PRIMARY_TRIPPED); executor.tick(1); assertThat(breaker.getState()).isEqualTo(PRIMARY_PROBE); breaker.requestComplete(false); assertThat(breaker.getState()).isEqualTo(SECONDARY_TRIPPED); executor.tick(SECONDARY_TRIP_TIME - 1); assertThat(breaker.getState()).isEqualTo(SECONDARY_TRIPPED); executor.tick(1); assertThat(breaker.getState()).isEqualTo(SECONDARY_PROBE); breaker.requestComplete(false); assertThat(breaker.getState()).isEqualTo(TERTIARY_TRIPPED); executor.tick(TERTIARY_TRIP_TIME - 1); assertThat(breaker.getState()).isEqualTo(TERTIARY_TRIPPED); executor.tick(1); assertThat(breaker.getState()).isEqualTo(TERTIARY_PROBE); breaker.requestComplete(false); assertThat(breaker.getState()).isEqualTo(TERTIARY_TRIPPED); executor.tick(TERTIARY_TRIP_TIME); assertThat(breaker.getState()).isEqualTo(TERTIARY_PROBE); breaker.requestComplete(true);
@Before public void setup() { breaker = new CircuitBreakerState(executor); }
@Before public void setup() { executor = new StubExecutor(2); breaker = new CircuitBreakerState(executor); }