if (node.getType() == NodeConnectivityType.UNAVAILABLE) { symmetric.put(nodeName, node); return; node.getConnectivity() if (adjNode.getType() == NodeConnectivityType.UNAVAILABLE){ if (!isLocalNode(node)){ newConnectivity.put(adjNodeName, node.getConnectionStatus(adjNodeName)); return; }); NodeConnectivity symmetricConnectivity = NodeConnectivity.builder() .endpoint(nodeName) .connectivity(ImmutableMap.copyOf(newConnectivity)) .type(node.getType()) .build();
/** * Get connection status between two nodes in the graph. * For instance: * {a: [{a: OK}, {b: FAILED}]} * {b: [{a: OK}, {b: FAILED}]} * getConnectionStatus("a", "b") -> FAILED * getConnectionStatus("b", "a") -> OK * * @param sourceNode source node * @param targetNode second node * @return connection status */ private ConnectionStatus getConnectionStatus(NodeConnectivity sourceNode, NodeConnectivity targetNode) { if (sourceNode == null || targetNode == null) { String errMsg = "Source or target node is null. Source: " + sourceNode + ", target: " + targetNode; throw new IllegalArgumentException(errMsg); } Set<NodeConnectivityType> types = EnumSet.of(sourceNode.getType(), targetNode.getType()); if (types.contains(NodeConnectivityType.UNAVAILABLE)) { return ConnectionStatus.FAILED; } return sourceNode.getConnectionStatus(targetNode.getEndpoint()); }
public static NodeState getUnavailableNodeState(String endpoint){ return new NodeState( NodeConnectivity.unavailable(endpoint), new HeartbeatTimestamp(Layout.INVALID_EPOCH, INVALID_HEARTBEAT_COUNTER), SequencerMetrics.UNKNOWN ); }
/** * See if the node is fully connected. * * @param endpoint local node name * @param unresponsiveNodes list of unresponsive nodes in the layout * @return local node rank */ public Optional<NodeRank> findFullyConnectedNode(String endpoint, List<String> unresponsiveNodes) { log.trace("Find responsive node. Unresponsive nodes: {}", unresponsiveNodes); NodeConnectivity node = getNodeConnectivity(endpoint); for (String adjacent : node.getConnectivity().keySet()) { //if adjacent node is unresponsive then then exclude it from fully connected graph if (unresponsiveNodes.contains(adjacent)) { continue; } NodeConnectivity adjacentNode = getNodeConnectivity(adjacent); //if adjacent node is unavailable then exclude it from fully connected graph if (adjacentNode.getType() == NodeConnectivityType.UNAVAILABLE) { continue; } if (adjacentNode.getConnectionStatus(endpoint) != ConnectionStatus.OK) { log.trace("Fully connected node not found"); return Optional.empty(); } } return Optional.of(new NodeRank(endpoint, getNodeConnectivity(endpoint).getConnected())); }
/** * Builds a new connectivity with unavailable state * @param endpoint node name * @return NodeConnectivity */ public static NodeConnectivity unavailable(String endpoint) { return NodeConnectivity.builder() .endpoint(endpoint) .type(NodeConnectivityType.UNAVAILABLE) .connectivity(ImmutableMap.of()) .build(); }
@Test public void testToSymmetricForThreeNodesWithUnavailableNodeB() { NodeConnectivity a = connectivity("a", ImmutableMap.of("a", OK, "b", FAILED, "c", FAILED)); NodeConnectivity b = unavailable("b"); NodeConnectivity c = connectivity("c", ImmutableMap.of("a", OK, "b", OK, "c", OK)); ClusterGraph graph = cluster("a", a, b, c); ClusterGraph symmetric = graph.toSymmetric(); NodeConnectivity symmetricNodeA = symmetric.getNodeConnectivity("a"); assertEquals(OK, symmetricNodeA.getConnectionStatus("a")); assertEquals(FAILED, symmetricNodeA.getConnectionStatus("b")); assertEquals(FAILED, symmetricNodeA.getConnectionStatus("c")); assertThatThrownBy(() -> symmetric.getNodeConnectivity("b").getConnectionStatus("a")) .isInstanceOf(IllegalStateException.class); assertEquals(FAILED, symmetric.getNodeConnectivity("c").getConnectionStatus("a")); assertEquals(OK, symmetric.getNodeConnectivity("c").getConnectionStatus("b")); assertEquals(OK, symmetric.getNodeConnectivity("c").getConnectionStatus("c")); }
private boolean isLocalNode(NodeConnectivity node) { return node.getEndpoint().equals(localNode); }
@Test public void testToSymmetricForTwoNodes() { NodeConnectivity a = connectivity("a", ImmutableMap.of("a", OK, "b", FAILED)); NodeConnectivity b = connectivity("b", ImmutableMap.of("a", OK, "b", OK)); ClusterGraph graph = cluster("a", a, b); ClusterGraph symmetric = graph.toSymmetric(); assertEquals(FAILED, graph.getNodeConnectivity("a").getConnectionStatus("b")); assertEquals(OK, graph.getNodeConnectivity("b").getConnectionStatus("a")); assertEquals(FAILED, symmetric.getNodeConnectivity("a").getConnectionStatus("b")); assertEquals(FAILED, symmetric.getNodeConnectivity("b").getConnectionStatus("a")); }
@Test public void testFindFullyConnectedResponsiveNodes() { ClusterGraph graph = cluster( "a", connectivity("a", ImmutableMap.of("a", OK, "b", FAILED, "c", OK)), unavailable("b"), connectivity("c", ImmutableMap.of("a", OK, "b", FAILED, "c", OK)) ); graph = graph.toSymmetric(); Optional<NodeRank> responsiveNode = graph.findFullyConnectedNode("c", Collections.singletonList("b")); assertTrue(responsiveNode.isPresent()); assertEquals(new NodeRank("c", 2), responsiveNode.get()); responsiveNode = graph.findFullyConnectedNode("c", Collections.singletonList("c")); assertTrue(responsiveNode.isPresent()); assertEquals("c", responsiveNode.get().getEndpoint()); assertEquals(2, responsiveNode.get().getNumConnections()); graph = cluster( "a", connectivity("a", ImmutableMap.of("a", OK, "b", FAILED, "c", OK)), connectivity("b", ImmutableMap.of("a", FAILED, "b", OK, "c", FAILED)), connectivity("c", ImmutableMap.of("a", OK, "b", FAILED, "c", OK)) ); graph = graph.toSymmetric(); responsiveNode = graph.findFullyConnectedNode("c", Collections.singletonList("b")); assertTrue(responsiveNode.isPresent()); assertEquals(new NodeRank("c", 2), responsiveNode.get()); }
@Test public void testDecisionMaker() { NodeConnectivity a = connectivity("a", ImmutableMap.of("a", OK, "b", OK, "c", OK)); NodeConnectivity b = connectivity("b", ImmutableMap.of("a", FAILED, "b", OK, "c", OK)); NodeConnectivity c = connectivity("c", ImmutableMap.of("a", OK, "b", FAILED, "c", OK)); ClusterGraph graph = cluster("a", a, b, c); Optional<NodeRank> decisionMaker = graph.toSymmetric().getDecisionMaker(); assertTrue(decisionMaker.isPresent()); assertEquals(decisionMaker.get(), new NodeRank("a", 2)); }
/** * See if cluster is ready. If cluster contains at least one node with state NOT_READY the cluster is not ready and * cluster state can't be used to find failures. * @return cluster status */ public boolean isReady(){ if (nodes.isEmpty()){ return false; } //if at least one node is not ready then entire cluster is not ready to provide correct information for (NodeState nodeState : nodes.values()) { if (nodeState.getConnectivity().getType() == NodeConnectivityType.NOT_READY){ return false; } } return true; } }
private NavigableSet<NodeRank> getNodeRanks() { NavigableSet<NodeRank> nodes = new TreeSet<>(); graph.keySet().forEach(node -> { int numConnected = graph.get(node).getConnected(); nodes.add(new NodeRank(node, numConnected)); }); return nodes; }
/** * Builds a new connectivity with NOT_READY state * @param endpoint node name * @return NodeConnectivity */ public static NodeConnectivity notReady(String endpoint) { return NodeConnectivity.builder() .endpoint(endpoint) .type(NodeConnectivityType.NOT_READY) .connectivity(ImmutableMap.of()) .build(); }
private ClusterState buildClusterState(NodeState... states) { Map<String, NodeState> graph = Arrays.stream(states) .collect(Collectors.toMap(state -> state.getConnectivity().getEndpoint(), Function.identity())); return ClusterState.builder() .nodes(ImmutableMap.copyOf(graph)) .build(); }
@Test public void testToSymmetricForThreeNodes() { NodeConnectivity a = connectivity("a", ImmutableMap.of("a", OK, "b", FAILED, "c", FAILED)); NodeConnectivity b = connectivity("b", ImmutableMap.of("a", OK, "b", OK, "c", OK)); NodeConnectivity c = connectivity("c", ImmutableMap.of("a", OK, "b", FAILED, "c", OK)); ClusterGraph symmetric = cluster("a", a, b, c).toSymmetric(); assertEquals(FAILED, symmetric.getNodeConnectivity("b").getConnectionStatus("a")); assertEquals(FAILED, symmetric.getNodeConnectivity("c").getConnectionStatus("a")); symmetric = cluster("b", a, b, c).toSymmetric(); assertEquals(FAILED, symmetric.getNodeConnectivity("b").getConnectionStatus("a")); assertEquals(FAILED, symmetric.getNodeConnectivity("c").getConnectionStatus("a")); symmetric = cluster("c", a, b, c).toSymmetric(); assertEquals(FAILED, symmetric.getNodeConnectivity("b").getConnectionStatus("a")); assertEquals(FAILED, symmetric.getNodeConnectivity("c").getConnectionStatus("a")); }
@Test public void testFailedNode() { NodeConnectivity a = connectivity("a", ImmutableMap.of("a", OK, "b", OK, "c", OK)); NodeConnectivity b = connectivity("b", ImmutableMap.of("a", FAILED, "b", OK, "c", OK)); NodeConnectivity c = connectivity("c", ImmutableMap.of("a", OK, "b", FAILED, "c", OK)); ClusterGraph graph = cluster("a", a, b, c); Optional<NodeRank> failedNode = graph.toSymmetric().findFailedNode(); assertTrue(failedNode.isPresent()); assertEquals(failedNode.get(), new NodeRank("b", 1)); }
/** * Factory method to build a node connectivity * @param endpoint node name * @param connectivity connectivity matrix * @return NodeConnectivity */ public static NodeConnectivity connectivity(String endpoint, ImmutableMap<String, ConnectionStatus> connectivity) { return NodeConnectivity.builder() .endpoint(endpoint) .type(NodeConnectivityType.CONNECTED) .connectivity(connectivity) .build(); }
nodeStates.put(localNodeState.getConnectivity().getEndpoint(), localNodeState);
/** * Helper method to build local {@link NodeState} based on pings * * @param allServers all servers in the cluster * @param allConnectedNodes all connected nodes in the cluster * @param epoch current epoch * @param sequencerMetrics metrics * @return local node state */ private NodeState getLocalNodeState(Set<String> allServers, Set<String> allConnectedNodes, long epoch, SequencerMetrics sequencerMetrics) { //Get connectivity status for a local node Map<String, ConnectionStatus> connectivity = new HashMap<>(); allServers.forEach(server -> { connectivity.put(server, ConnectionStatus.fromBool(allConnectedNodes.contains(server))); }); NodeConnectivity localConnectivity = NodeConnectivity.connectivity( localEndpoint, ImmutableMap.copyOf(connectivity) ); return NodeState.builder() .connectivity(localConnectivity) .heartbeat(new HeartbeatTimestamp(epoch, heartbeatCounter.incrementHeartbeat())) .sequencerMetrics(sequencerMetrics) .build(); }
private NodeState unavailableNodeState(String endpoint) { return new NodeState( unavailable(endpoint), new HeartbeatTimestamp(Layout.INVALID_EPOCH, 0), SequencerMetrics.UNKNOWN ); } }