private NavigableSet<NodeRank> getNodeRanks() { NavigableSet<NodeRank> nodes = new TreeSet<>(); graph.keySet().forEach(node -> { int numConnected = graph.get(node).getConnected(); nodes.add(new NodeRank(node, numConnected)); }); return nodes; }
/** * See if the node is fully connected. * * @param endpoint local node name * @param unresponsiveNodes list of unresponsive nodes in the layout * @return local node rank */ public Optional<NodeRank> findFullyConnectedNode(String endpoint, List<String> unresponsiveNodes) { log.trace("Find responsive node. Unresponsive nodes: {}", unresponsiveNodes); NodeConnectivity node = getNodeConnectivity(endpoint); for (String adjacent : node.getConnectivity().keySet()) { //if adjacent node is unresponsive then then exclude it from fully connected graph if (unresponsiveNodes.contains(adjacent)) { continue; } NodeConnectivity adjacentNode = getNodeConnectivity(adjacent); //if adjacent node is unavailable then exclude it from fully connected graph if (adjacentNode.getType() == NodeConnectivityType.UNAVAILABLE) { continue; } if (adjacentNode.getConnectionStatus(endpoint) != ConnectionStatus.OK) { log.trace("Fully connected node not found"); return Optional.empty(); } } return Optional.of(new NodeRank(endpoint, getNodeConnectivity(endpoint).getConnected())); }
@Test public void testDecisionMaker() { NodeConnectivity a = connectivity("a", ImmutableMap.of("a", OK, "b", OK, "c", OK)); NodeConnectivity b = connectivity("b", ImmutableMap.of("a", FAILED, "b", OK, "c", OK)); NodeConnectivity c = connectivity("c", ImmutableMap.of("a", OK, "b", FAILED, "c", OK)); ClusterGraph graph = cluster("a", a, b, c); Optional<NodeRank> decisionMaker = graph.toSymmetric().getDecisionMaker(); assertTrue(decisionMaker.isPresent()); assertEquals(decisionMaker.get(), new NodeRank("a", 2)); }
@Test public void testFailedNode() { NodeConnectivity a = connectivity("a", ImmutableMap.of("a", OK, "b", OK, "c", OK)); NodeConnectivity b = connectivity("b", ImmutableMap.of("a", FAILED, "b", OK, "c", OK)); NodeConnectivity c = connectivity("c", ImmutableMap.of("a", OK, "b", FAILED, "c", OK)); ClusterGraph graph = cluster("a", a, b, c); Optional<NodeRank> failedNode = graph.toSymmetric().findFailedNode(); assertTrue(failedNode.isPresent()); assertEquals(failedNode.get(), new NodeRank("b", 1)); }
@Test public void testFailedServer_disconnected_c() { CompleteGraphAdvisor advisor = new CompleteGraphAdvisor("a"); ClusterState clusterState = buildClusterState( nodeState("a", OK, OK, FAILED), nodeState("b", OK, OK, FAILED), unavailable("c") ); List<String> unresponsiveServers = new ArrayList<>(); Optional<NodeRank> failedServer = advisor.failedServer(clusterState, unresponsiveServers); assertTrue(failedServer.isPresent()); assertEquals(new NodeRank("c", 0), failedServer.get()); }
@Test public void testHealedServer() { CompleteGraphAdvisor advisor = new CompleteGraphAdvisor("c"); ClusterState clusterState = buildClusterState( nodeState("a", OK, FAILED, OK), nodeState("b", FAILED, OK, OK), nodeState("c", OK, OK, OK) ); List<String> unresponsiveServers = new ArrayList<>(); unresponsiveServers.add("c"); Optional<NodeRank> healedServer = advisor.healedServer(clusterState, unresponsiveServers); assertTrue(healedServer.isPresent()); assertEquals(new NodeRank("c", clusterState.size()), healedServer.get()); }
@Test public void testFailedServer_asymmetricFailureBetween_b_c() { CompleteGraphAdvisor advisor = new CompleteGraphAdvisor("a"); ClusterState clusterState = buildClusterState( nodeState("a", OK, OK, OK), nodeState("b", OK, OK, FAILED), nodeState("c", OK, FAILED, OK) ); List<String> unresponsiveServers = new ArrayList<>(); Optional<NodeRank> failedServer = advisor.failedServer(clusterState, unresponsiveServers); assertTrue(failedServer.isPresent()); assertEquals(new NodeRank("c", 2), failedServer.get()); }
@Test public void testFindFullyConnectedResponsiveNodes() { ClusterGraph graph = cluster( "a", connectivity("a", ImmutableMap.of("a", OK, "b", FAILED, "c", OK)), unavailable("b"), connectivity("c", ImmutableMap.of("a", OK, "b", FAILED, "c", OK)) ); graph = graph.toSymmetric(); Optional<NodeRank> responsiveNode = graph.findFullyConnectedNode("c", Collections.singletonList("b")); assertTrue(responsiveNode.isPresent()); assertEquals(new NodeRank("c", 2), responsiveNode.get()); responsiveNode = graph.findFullyConnectedNode("c", Collections.singletonList("c")); assertTrue(responsiveNode.isPresent()); assertEquals("c", responsiveNode.get().getEndpoint()); assertEquals(2, responsiveNode.get().getNumConnections()); graph = cluster( "a", connectivity("a", ImmutableMap.of("a", OK, "b", FAILED, "c", OK)), connectivity("b", ImmutableMap.of("a", FAILED, "b", OK, "c", FAILED)), connectivity("c", ImmutableMap.of("a", OK, "b", FAILED, "c", OK)) ); graph = graph.toSymmetric(); responsiveNode = graph.findFullyConnectedNode("c", Collections.singletonList("b")); assertTrue(responsiveNode.isPresent()); assertEquals(new NodeRank("c", 2), responsiveNode.get()); }
/** * B node believes that everyone disconnected, but actually it's B disconnected. * Make a decision to exclude C. */ @Test public void testFailedServer_allDisconnected_from_b_perspective() { CompleteGraphAdvisor advisor = new CompleteGraphAdvisor("b"); ClusterState clusterState = buildClusterState( unavailable("a"), nodeState("b", OK, OK, OK), unavailable("c") ); List<String> unresponsiveServers = new ArrayList<>(); Optional<NodeRank> failedServer = advisor.failedServer(clusterState, unresponsiveServers); assertTrue(failedServer.isPresent()); assertEquals(new NodeRank("c", 0), failedServer.get()); }
assertEquals(new NodeRank("c", 2), nodeAFailedServer.get());