static CassandraRequestExceptionHandler withNoBackoffForTest( Supplier<Integer> maxTriesSameHost, Supplier<Integer> maxTriesTotal, Blacklist blacklist) { return new CassandraRequestExceptionHandler(maxTriesSameHost, maxTriesTotal, blacklist); }
@SuppressWarnings("unchecked") <K extends Exception> void handleExceptionFromRequest( RetryableCassandraRequest<?, K> req, InetSocketAddress hostTried, Exception ex) throws K { if (!isRetryable(ex)) { throw (K) ex; } RequestExceptionHandlerStrategy strategy = getStrategy(); req.triedOnHost(hostTried); req.registerException(ex); int numberOfAttempts = req.getNumberOfAttempts(); int numberOfAttemptsOnHost = req.getNumberOfAttemptsOnHost(hostTried); if (numberOfAttempts >= maxTriesTotal.get()) { throw logAndThrowException(numberOfAttempts, ex, req); } if (shouldBlacklist(ex, numberOfAttemptsOnHost)) { blacklist.add(hostTried); } logNumberOfAttempts(ex, numberOfAttempts); handleBackoff(req, hostTried, ex, strategy); handleRetryOnDifferentHosts(req, hostTried, ex, strategy); }
@Test public void connectionExceptionRetriesOnDifferentHostAfterSufficientRetriesDefault() { for (Exception ex : CONNECTION_EXCEPTIONS) { assertFalse(String.format("Exception %s should not retry on different host", ex), handlerLegacy.shouldRetryOnDifferentHost(ex, MAX_RETRIES_PER_HOST - 1, handlerLegacy.getStrategy())); } for (Exception ex : CONNECTION_EXCEPTIONS) { assertTrue(String.format("Exception %s should retry on different host", ex), handlerLegacy.shouldRetryOnDifferentHost(ex, MAX_RETRIES_PER_HOST, handlerLegacy.getStrategy())); } }
@Override public boolean shouldBackoff(Exception ex) { return isConnectionException(ex) || isIndicativeOfCassandraLoad(ex); }
@VisibleForTesting static boolean isRetryable(Exception ex) { return isConnectionException(ex) || isTransientException(ex) || isIndicativeOfCassandraLoad(ex) || isFastFailoverException(ex); }
@Test public void cassandraLoadExceptionsShouldBackoffDefault() { for (Exception ex : INDICATIVE_OF_CASSANDRA_LOAD_EXCEPTIONS) { assertTrue(String.format("Exception %s should backoff", ex), handlerLegacy.shouldBackoff(ex, handlerLegacy.getStrategy())); } }
@Test public void changingHandlerModeHasNoEffectWithoutGetStrategy() { Exception ex = Iterables.get(TRANSIENT_EXCEPTIONS, 0); CassandraRequestExceptionHandler handler = new CassandraRequestExceptionHandler( () -> MAX_RETRIES_PER_HOST, () -> MAX_RETRIES_TOTAL, this::mutableMode, new Blacklist(config)); CassandraRequestExceptionHandler.RequestExceptionHandlerStrategy conservativeStrategy = handler.getStrategy(); assertTrue(handler.shouldBackoff(ex, handler.getStrategy())); flipMode(); assertTrue(handler.shouldBackoff(ex, conservativeStrategy)); assertFalse(handler.shouldBackoff(ex, handler.getStrategy())); }
static boolean isConnectionException(Throwable ex) { return ex != null // tcp socket timeout, possibly indicating network flake, long GC, or restarting server. && (ex instanceof SocketTimeoutException || ex instanceof CassandraClientFactory.ClientCreationFailedException || isConnectionException(ex.getCause())); }
@Test public void retryableExceptionsAreRetryableConservative() { for (Exception ex : ALL_EXCEPTIONS) { assertTrue(String.format("Exception %s should be retryable", ex), handlerConservative.isRetryable(ex)); } assertFalse("RuntimeException is not retryable", handlerConservative.isRetryable(new RuntimeException())); }
private <K extends Exception> void handleRetryOnDifferentHosts(RetryableCassandraRequest<?, K> req, InetSocketAddress hostTried, Exception ex, RequestExceptionHandlerStrategy strategy) { if (shouldRetryOnDifferentHost(ex, req.getNumberOfAttemptsOnHost(hostTried), strategy)) { log.info("Retrying a query intended for host {} on a different host.", SafeArg.of("hostName", CassandraLogHelper.host(hostTried))); req.giveUpOnPreferredHost(); } }
@Test public void connectionExceptionsWithSufficientAttemptsShouldBlacklistConservative() { for (Exception ex : CONNECTION_EXCEPTIONS) { assertFalse("MAX_RETRIES_PER_HOST - 1 attempts should not blacklist", handlerConservative.shouldBlacklist(ex, MAX_RETRIES_PER_HOST - 1)); } for (Exception ex : CONNECTION_EXCEPTIONS) { assertTrue(String.format("MAX_RETRIES_PER_HOST attempts with exception %s should blacklist", ex), handlerConservative.shouldBlacklist(ex, MAX_RETRIES_PER_HOST)); } Exception ffException = Iterables.get(FAST_FAILOVER_EXCEPTIONS, 0); assertFalse(String.format("Exception %s should not blacklist", ffException), handlerConservative.shouldBlacklist(ffException, MAX_RETRIES_PER_HOST)); }
private <K extends Exception> void handleBackoff(RetryableCassandraRequest<?, K> req, InetSocketAddress hostTried, Exception ex, RequestExceptionHandlerStrategy strategy) { if (!shouldBackoff(ex, strategy)) { return; } long backOffPeriod = strategy.getBackoffPeriod(req.getNumberOfAttemptsOnHost(hostTried)); log.info("Retrying a query, {}, with backoff of {}ms, intended for host {}.", UnsafeArg.of("queryString", req.getFunction().toString()), SafeArg.of("sleepDuration", backOffPeriod), SafeArg.of("hostName", CassandraLogHelper.host(hostTried))); try { Thread.sleep(backOffPeriod); } catch (InterruptedException i) { Thread.currentThread().interrupt(); throw new RuntimeException(i); } }
@Override public <V, K extends Exception> V runWithRetryOnHost( InetSocketAddress specifiedHost, FunctionCheckedException<CassandraClient, V, K> fn) throws K { RetryableCassandraRequest<V, K> req = new RetryableCassandraRequest<>(specifiedHost, fn); while (true) { if (log.isTraceEnabled()) { log.trace("Running function on host {}.", SafeArg.of("host", CassandraLogHelper.host(req.getPreferredHost()))); } CassandraClientPoolingContainer hostPool = getPreferredHostOrFallBack(req); try { V response = runWithPooledResourceRecordingMetrics(hostPool, req.getFunction()); removeFromBlacklistAfterResponse(hostPool.getHost()); return response; } catch (Exception ex) { exceptionHandler.handleExceptionFromRequest(req, hostPool.getHost(), ex); } } }
@Test public void connectionExceptionsShouldBackoffDefault() { for (Exception ex : CONNECTION_EXCEPTIONS) { assertTrue(String.format("Exception %s should backoff", ex), handlerLegacy.shouldBackoff(ex, handlerLegacy.getStrategy())); } }
@VisibleForTesting static boolean isRetryable(Exception ex) { return isConnectionException(ex) || isTransientException(ex) || isIndicativeOfCassandraLoad(ex) || isFastFailoverException(ex); }
@Override public boolean shouldBackoff(Exception ex) { return isConnectionException(ex) || isIndicativeOfCassandraLoad(ex); }
@VisibleForTesting boolean shouldBlacklist(Exception ex, int numberOfAttempts) { return isConnectionException(ex) && numberOfAttempts >= maxTriesSameHost.get(); }
@Test public void retryableExceptionsAreRetryableDefault() { for (Exception ex : ALL_EXCEPTIONS) { assertTrue(String.format("Exception %s should be retryable", ex), handlerLegacy.isRetryable(ex)); } assertFalse("RuntimeException is not retryable", handlerLegacy.isRetryable(new RuntimeException())); }
private <K extends Exception> void handleRetryOnDifferentHosts(RetryableCassandraRequest<?, K> req, InetSocketAddress hostTried, Exception ex, RequestExceptionHandlerStrategy strategy) { if (shouldRetryOnDifferentHost(ex, req.getNumberOfAttemptsOnHost(hostTried), strategy)) { log.info("Retrying a query intended for host {} on a different host.", SafeArg.of("hostName", CassandraLogHelper.host(hostTried))); req.giveUpOnPreferredHost(); } }
@Test public void connectionExceptionsWithSufficientAttemptsShouldBlacklistDefault() { for (Exception ex : CONNECTION_EXCEPTIONS) { assertFalse("MAX_RETRIES_PER_HOST - 1 attempts should not blacklist", handlerLegacy.shouldBlacklist(ex, MAX_RETRIES_PER_HOST - 1)); } for (Exception ex : CONNECTION_EXCEPTIONS) { assertTrue(String.format("MAX_RETRIES_PER_HOST attempts with exception %s should blacklist", ex), handlerLegacy.shouldBlacklist(ex, MAX_RETRIES_PER_HOST)); } Exception ffException = Iterables.get(FAST_FAILOVER_EXCEPTIONS, 0); assertFalse(String.format("Exception %s should not blacklist", ffException), handlerLegacy.shouldBlacklist(ffException, MAX_RETRIES_PER_HOST)); }