private ClusterAvailabilityStatus getStatusByRunningOperationsOnEachHost() { int countUnreachableNodes = 0; for (InetSocketAddress host : clientPool.getCurrentPools().keySet()) { try { clientPool.runOnHost(host, CassandraVerifier.healthCheck); if (!partitionerIsValid(host)) { return ClusterAvailabilityStatus.TERMINAL; } } catch (Exception e) { countUnreachableNodes++; } } return getNodeAvailabilityStatus(countUnreachableNodes); }
private void verifyNumberOfAttemptsOnHost(InetSocketAddress host, CassandraClientPool cassandraClientPool, int numAttempts) { Mockito.verify(cassandraClientPool.getCurrentPools().get(host), Mockito.times(numAttempts)) .runWithPooledResource( Mockito.<FunctionCheckedException<CassandraClient, Object, RuntimeException>>any()); }
void inPool(CassandraClientPool cassandraClientPool) { CassandraClientPoolingContainer container = mock(CassandraClientPoolingContainer.class); when(container.getHost()).thenReturn(address); try { OngoingStubbing<Object> stubbing = when(container.runWithPooledResource( Mockito.<FunctionCheckedException<CassandraClient, Object, Exception>>any())); for (Exception ex : exceptions) { stubbing = stubbing.thenThrow(ex); } if (returnsValue) { stubbing.thenReturn("Response"); } } catch (Exception ex) { throw new RuntimeException(ex); } cassandraClientPool.getCurrentPools().put(address, container); } }
@Test public void resilientToRollingRestarts() { CassandraClientPool cassandraClientPool = clientPoolWithServersInCurrentPool(ImmutableSet.of(HOST_1, HOST_2)); AtomicReference<InetSocketAddress> downHost = new AtomicReference<>(HOST_1); cassandraClientPool.getCurrentPools().values().forEach(pool -> setConditionalTimeoutFailureForHost( pool, container -> container.getHost().equals(downHost.get()))); runNoopWithRetryOnHost(HOST_1, cassandraClientPool); assertThat(blacklist.contains(HOST_1), is(true)); downHost.set(HOST_2); runNoopWithRetryOnHost(HOST_2, cassandraClientPool); assertThat(blacklist.contains(HOST_1), is(false)); }
@Test public void successfulRequestCausesHostToBeRemovedFromBlacklist() { CassandraClientPool cassandraClientPool = clientPoolWithServersInCurrentPool(ImmutableSet.of(HOST_1)); CassandraClientPoolingContainer container = cassandraClientPool.getCurrentPools().get(HOST_1); AtomicBoolean fail = new AtomicBoolean(true); setConditionalTimeoutFailureForHost(container, unused -> fail.get()); assertThatThrownBy(() -> runNoopWithRetryOnHost(HOST_1, cassandraClientPool)) .isInstanceOf(AtlasDbDependencyException.class); assertThat(blacklist.contains(HOST_1), is(true)); fail.set(false); runNoopWithRetryOnHost(HOST_1, cassandraClientPool); assertThat(blacklist.contains(HOST_1), is(false)); }
@Test public void testBlacklistMetrics() { CassandraClientPool cassandraClientPool = clientPoolWithServersInCurrentPool(ImmutableSet.of(HOST_1, HOST_2)); CassandraClientPoolingContainer container = cassandraClientPool.getCurrentPools().get(HOST_1); runNoopWithRetryOnHost(HOST_1, cassandraClientPool); verifyBlacklistMetric(0); setFailureModeForHost(container, new SocketTimeoutException()); runNoopWithRetryOnHost(HOST_1, cassandraClientPool); verifyBlacklistMetric(1); }
private ClusterAvailabilityStatus getStatusByRunningOperationsOnEachHost() { int countUnreachableNodes = 0; for (InetSocketAddress host : clientPool.getCurrentPools().keySet()) { try { clientPool.runOnHost(host, CassandraVerifier.healthCheck); if (!partitionerIsValid(host)) { return ClusterAvailabilityStatus.TERMINAL; } } catch (Exception e) { countUnreachableNodes++; } } return getNodeAvailabilityStatus(countUnreachableNodes); }