private boolean shouldSwitchNode(ExceptionRetryBehaviour retryBehaviour, long failures) { return retryBehaviour.shouldBackoffAndTryOtherNodes() || (!retryBehaviour.shouldRetryInfinitelyManyTimes() && failures >= failuresBeforeSwitching); }
@Test public void retryIndefinitelyOnSameNodeShouldRetryInfinitelyManyTimes() { ExceptionRetryBehaviour behaviour = ExceptionRetryBehaviour.RETRY_INDEFINITELY_ON_SAME_NODE; assertThat(behaviour.shouldRetryInfinitelyManyTimes()).isTrue(); }
@Test public void retryOnOtherNodesShouldRetryInfinitelyManyTimes() { ExceptionRetryBehaviour behaviour = ExceptionRetryBehaviour.RETRY_ON_OTHER_NODE; assertThat(behaviour.shouldRetryInfinitelyManyTimes()).isTrue(); }
@Test public void retryOnSameNodeShouldRetryFinitelyManyTimes() { ExceptionRetryBehaviour behaviour = ExceptionRetryBehaviour.RETRY_ON_SAME_NODE; assertThat(behaviour.shouldRetryInfinitelyManyTimes()).isFalse(); }
@Override public void continueOrPropagate(RetryableException ex) { ExceptionRetryBehaviour retryBehaviour = ExceptionRetryBehaviour.getRetryBehaviourForException(ex); synchronized (this) { // Only fail over if this failure was to the current server. // This means that no one on another thread has failed us over already. if (mostRecentServerIndex.get() != null && mostRecentServerIndex.get() == failoverCount.get()) { long failures = failuresSinceLastSwitch.incrementAndGet(); if (shouldSwitchNode(retryBehaviour, failures)) { failoverToNextNode(retryBehaviour); } else if (retryBehaviour.shouldRetryInfinitelyManyTimes()) { failuresSinceLastSwitch.set(0); } } } checkAndHandleFailure(ex); if (retryBehaviour.shouldBackoffAndTryOtherNodes()) { int numFailovers = failoverCount.get(); if (numFailovers > 0 && numFailovers % servers.size() == 0) { // We implement some randomness around the expected value of BACKOFF_BEFORE_ROUND_ROBIN_RETRY_MILLIS. // Even though this is not exponential backoff, should be enough to avoid a thundering herd problem. long pauseTimeWithJitter = ThreadLocalRandom.current() .nextLong(BACKOFF_BEFORE_ROUND_ROBIN_RETRY_MILLIS / 2, (BACKOFF_BEFORE_ROUND_ROBIN_RETRY_MILLIS * 3) / 2); pauseForBackoff(ex, pauseTimeWithJitter); } } else { pauseForBackoff(ex); } }
private boolean shouldSwitchNode(ExceptionRetryBehaviour retryBehaviour, long failures) { return retryBehaviour.shouldBackoffAndTryOtherNodes() || (!retryBehaviour.shouldRetryInfinitelyManyTimes() && failures >= failuresBeforeSwitching); }
@Override public void continueOrPropagate(RetryableException ex) { ExceptionRetryBehaviour retryBehaviour = ExceptionRetryBehaviour.getRetryBehaviourForException(ex); synchronized (this) { // Only fail over if this failure was to the current server. // This means that no one on another thread has failed us over already. if (mostRecentServerIndex.get() != null && mostRecentServerIndex.get() == failoverCount.get()) { long failures = failuresSinceLastSwitch.incrementAndGet(); if (shouldSwitchNode(retryBehaviour, failures)) { failoverToNextNode(retryBehaviour); } else if (retryBehaviour.shouldRetryInfinitelyManyTimes()) { failuresSinceLastSwitch.set(0); } } } checkAndHandleFailure(ex); if (retryBehaviour.shouldBackoffAndTryOtherNodes()) { int numFailovers = failoverCount.get(); if (numFailovers > 0 && numFailovers % servers.size() == 0) { // We implement some randomness around the expected value of BACKOFF_BEFORE_ROUND_ROBIN_RETRY_MILLIS. // Even though this is not exponential backoff, should be enough to avoid a thundering herd problem. long pauseTimeWithJitter = ThreadLocalRandom.current() .nextLong(BACKOFF_BEFORE_ROUND_ROBIN_RETRY_MILLIS / 2, (BACKOFF_BEFORE_ROUND_ROBIN_RETRY_MILLIS * 3) / 2); pauseForBackoff(ex, pauseTimeWithJitter); } } else { pauseForBackoff(ex); } }