private void removeFromBlacklistAfterResponse(InetSocketAddress host) { if (blacklist.contains(host)) { blacklist.remove(host); log.info("Added host {} back into the pool after receiving a successful response", SafeArg.of("host", CassandraLogHelper.host(host))); } }
@Test public void successfulRequestCausesHostToBeRemovedFromBlacklist() { CassandraClientPool cassandraClientPool = clientPoolWithServersInCurrentPool(ImmutableSet.of(HOST_1)); CassandraClientPoolingContainer container = cassandraClientPool.getCurrentPools().get(HOST_1); AtomicBoolean fail = new AtomicBoolean(true); setConditionalTimeoutFailureForHost(container, unused -> fail.get()); assertThatThrownBy(() -> runNoopWithRetryOnHost(HOST_1, cassandraClientPool)) .isInstanceOf(AtlasDbDependencyException.class); assertThat(blacklist.contains(HOST_1), is(true)); fail.set(false); runNoopWithRetryOnHost(HOST_1, cassandraClientPool); assertThat(blacklist.contains(HOST_1), is(false)); }
@Test public void resilientToRollingRestarts() { CassandraClientPool cassandraClientPool = clientPoolWithServersInCurrentPool(ImmutableSet.of(HOST_1, HOST_2)); AtomicReference<InetSocketAddress> downHost = new AtomicReference<>(HOST_1); cassandraClientPool.getCurrentPools().values().forEach(pool -> setConditionalTimeoutFailureForHost( pool, container -> container.getHost().equals(downHost.get()))); runNoopWithRetryOnHost(HOST_1, cassandraClientPool); assertThat(blacklist.contains(HOST_1), is(true)); downHost.set(HOST_2); runNoopWithRetryOnHost(HOST_2, cassandraClientPool); assertThat(blacklist.contains(HOST_1), is(false)); }
private <V, K extends Exception> CassandraClientPoolingContainer getPreferredHostOrFallBack( RetryableCassandraRequest<V, K> req) { CassandraClientPoolingContainer hostPool = cassandra.getPools().get(req.getPreferredHost()); if (blacklist.contains(req.getPreferredHost()) || hostPool == null || req.shouldGiveUpOnPreferredHost()) { InetSocketAddress previousHost = hostPool == null ? req.getPreferredHost() : hostPool.getHost(); Optional<CassandraClientPoolingContainer> hostPoolCandidate = cassandra.getRandomGoodHostForPredicate(address -> !req.alreadyTriedOnHost(address)); hostPool = hostPoolCandidate.orElseGet(cassandra::getRandomGoodHost); log.warn("Randomly redirected a query intended for host {} to {}.", SafeArg.of("previousHost", CassandraLogHelper.host(previousHost)), SafeArg.of("randomHost", CassandraLogHelper.host(hostPool.getHost()))); } return hostPool; }
@Test public void attemptsShouldBeCountedPerHost() { CassandraClientPoolImpl cassandraClientPool = CassandraClientPoolImpl.createImplForTest( MetricsManagers.of(metricRegistry, taggedMetricRegistry), config, CassandraClientPoolImpl.StartupChecks.DO_NOT_RUN, blacklist); host(HOST_1).throwsException(new SocketTimeoutException()) .throwsException(new InvalidRequestException()) .inPool(cassandraClientPool); host(HOST_2).throwsException(new SocketTimeoutException()) .inPool(cassandraClientPool); runNoopWithRetryOnHost(HOST_1, cassandraClientPool); assertThat(blacklist.contains(HOST_2), is(false)); }
private void removeFromBlacklistAfterResponse(InetSocketAddress host) { if (blacklist.contains(host)) { blacklist.remove(host); log.info("Added host {} back into the pool after receiving a successful response", SafeArg.of("host", CassandraLogHelper.host(host))); } }
private <V, K extends Exception> CassandraClientPoolingContainer getPreferredHostOrFallBack( RetryableCassandraRequest<V, K> req) { CassandraClientPoolingContainer hostPool = cassandra.getPools().get(req.getPreferredHost()); if (blacklist.contains(req.getPreferredHost()) || hostPool == null || req.shouldGiveUpOnPreferredHost()) { InetSocketAddress previousHost = hostPool == null ? req.getPreferredHost() : hostPool.getHost(); Optional<CassandraClientPoolingContainer> hostPoolCandidate = cassandra.getRandomGoodHostForPredicate(address -> !req.alreadyTriedOnHost(address)); hostPool = hostPoolCandidate.orElseGet(cassandra::getRandomGoodHost); log.warn("Randomly redirected a query intended for host {} to {}.", SafeArg.of("previousHost", CassandraLogHelper.host(previousHost)), SafeArg.of("randomHost", CassandraLogHelper.host(hostPool.getHost()))); } return hostPool; }