public NodeConnectionStatus(final NodeConnectionStatus status) { this(status.getNodeIdentifier(), status.getState(), status.getOffloadCode(), status.getDisconnectCode(), status.getReason(), status.getConnectionRequestTime()); }
@Override public void setLocalNodeIdentifier(final NodeIdentifier nodeId) { if (nodeId == null || nodeId.equals(this.nodeId)) { return; } this.nodeId = nodeId; nodeStatuses.computeIfAbsent(nodeId, id -> new NodeConnectionStatus(id, DisconnectionCode.NOT_YET_CONNECTED)); eventListeners.forEach(listener -> listener.onLocalNodeIdentifierSet(nodeId)); }
@Override public String toString() { final StringBuilder sb = new StringBuilder(); final NodeConnectionState state = getState(); sb.append("NodeConnectionStatus[nodeId=").append(nodeId).append(", state=").append(state); if (state == NodeConnectionState.OFFLOADED || state == NodeConnectionState.OFFLOADING) { sb.append(", Offload Code=").append(getOffloadCode()).append(", Offload Reason=").append(getReason()); } if (state == NodeConnectionState.DISCONNECTED || state == NodeConnectionState.DISCONNECTING) { sb.append(", Disconnect Code=").append(getDisconnectCode()).append(", Disconnect Reason=").append(getReason()); } sb.append(", updateId=").append(getUpdateIdentifier()); sb.append("]"); return sb.toString(); }
@Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (!(obj instanceof NodeConnectionStatus)) { return false; } NodeConnectionStatus other = (NodeConnectionStatus) obj; return Objects.deepEquals(getNodeIdentifier(), other.getNodeIdentifier()) && Objects.deepEquals(getState(), other.getState()); } }
private String summarizeStatusChange(final NodeConnectionStatus oldStatus, final NodeConnectionStatus status) { final StringBuilder sb = new StringBuilder(); if (oldStatus == null || status.getState() != oldStatus.getState()) { sb.append("Node Status changed from ").append(oldStatus == null ? "[Unknown Node]" : oldStatus.getState().toString()).append(" to ").append(status.getState().toString()); if (status.getReason() != null) { sb.append(" due to ").append(status.getReason()); } else if (status.getDisconnectCode() != null) { sb.append(" due to ").append(status.getDisconnectCode().toString()); } } return sb.toString(); }
public boolean isConnected() { rwLock.readLock().lock(); try { return connectionStatus != null && connectionStatus.getState() == NodeConnectionState.CONNECTED; } finally { rwLock.readLock().unlock(); } }
void updateNodeStatus(final NodeConnectionStatus status, final boolean waitForCoordinator) { final NodeIdentifier nodeId = status.getNodeIdentifier(); // In this case, we are using nodeStatuses.put() instead of getting the current value and // comparing that to the new value and using the one with the largest update id. This is because // this method is called when something occurs that causes this node to change the status of the // node in question. We only use comparisons against the current value when we receive an update // about a node status from a different node, since those may be received out-of-order. final NodeConnectionStatus currentStatus = updateNodeStatus(nodeId, status); final NodeConnectionState currentState = currentStatus == null ? null : currentStatus.getState(); logger.info("Status of {} changed from {} to {}", nodeId, currentStatus, status); logger.debug("State of cluster nodes is now {}", nodeStatuses); latestUpdateId.updateAndGet(curVal -> Math.max(curVal, status.getUpdateIdentifier())); if (currentState == null || currentState != status.getState()) { final boolean notifyAllNodes = isActiveClusterCoordinator(); if (notifyAllNodes) { logger.debug("Notifying all nodes that status changed from {} to {}", currentStatus, status); } else { logger.debug("Notifying cluster coordinator that node status changed from {} to {}", currentStatus, status); } notifyOthersOfNodeStatusChange(status, notifyAllNodes, waitForCoordinator); } else { logger.debug("Not notifying other nodes that status changed because previous state of {} is same as new state of {}", currentState, status.getState()); } }
final NodeConnectionState connectionState = connectionStatus.getState(); if (heartbeat.getConnectionStatus().getState() != NodeConnectionState.CONNECTED && connectionState == NodeConnectionState.CONNECTED) { + "though the Cluster Coordinator thought it was (node claimed state was " + heartbeat.getConnectionStatus().getState() + "). Marking as Disconnected and requesting that Node reconnect to cluster"); clusterCoordinator.requestNodeConnect(nodeId, null); final DisconnectionCode disconnectionCode = connectionStatus.getDisconnectCode(); clusterCoordinator.requestNodeDisconnect(nodeId, disconnectionCode, connectionStatus.getReason()); removeHeartbeat(nodeId); break; if (NodeConnectionState.DISCONNECTING == connectionStatus.getState()) { final Long connectionRequestTime = connectionStatus.getConnectionRequestTime(); if (connectionRequestTime != null && heartbeat.getTimestamp() < connectionRequestTime) { clusterCoordinator.reportEvent(nodeId, Severity.INFO, "Received heartbeat but ignoring because it was reported before the node was last asked to reconnect.");
private List<NodeConnectionStatus> getUpdatedStatuses(final List<NodeConnectionStatus> nodeStatusList) { // Map node's statuses by NodeIdentifier for quick & easy lookup final Map<NodeIdentifier, NodeConnectionStatus> nodeStatusMap = nodeStatusList.stream() .collect(Collectors.toMap(status -> status.getNodeIdentifier(), Function.identity())); // Check if our connection status is the same for each Node Identifier and if not, add our version of the status // to a List of updated statuses. final List<NodeConnectionStatus> currentStatuses = clusterCoordinator.getConnectionStatuses(); final List<NodeConnectionStatus> updatedStatuses = new ArrayList<>(); for (final NodeConnectionStatus currentStatus : currentStatuses) { final NodeConnectionStatus nodeStatus = nodeStatusMap.get(currentStatus.getNodeIdentifier()); if (!currentStatus.equals(nodeStatus)) { updatedStatuses.add(currentStatus); } } // If the node has any statuses that we do not have, add a REMOVED status to the update list final Set<NodeIdentifier> nodeIds = currentStatuses.stream().map(status -> status.getNodeIdentifier()).collect(Collectors.toSet()); for (final NodeConnectionStatus nodeStatus : nodeStatusList) { if (!nodeIds.contains(nodeStatus.getNodeIdentifier())) { updatedStatuses.add(new NodeConnectionStatus(nodeStatus.getNodeIdentifier(), NodeConnectionState.REMOVED, null)); } } logger.debug("\n\nCalculated diff between current cluster status and node cluster status as follows:\nNode: {}\nSelf: {}\nDifference: {}\n\n", nodeStatusList, currentStatuses, updatedStatuses); return updatedStatuses; }
private void handleNodeStatusChange(final NodeStatusChangeMessage statusChangeMessage) { final NodeConnectionStatus updatedStatus = statusChangeMessage.getNodeConnectionStatus(); final NodeIdentifier nodeId = statusChangeMessage.getNodeId(); logger.debug("Handling request {}", statusChangeMessage); final NodeConnectionStatus oldStatus = nodeStatuses.get(statusChangeMessage.getNodeId()); // Either remove the value from the map or update the map depending on the connection state if (statusChangeMessage.getNodeConnectionStatus().getState() == NodeConnectionState.REMOVED) { if (removeNodeConditionally(nodeId, oldStatus)) { storeState(); } } else { updateNodeStatus(nodeId, updatedStatus); } logger.info("Status of {} changed from {} to {}", statusChangeMessage.getNodeId(), oldStatus, updatedStatus); logger.debug("State of cluster nodes is now {}", nodeStatuses); final NodeConnectionStatus status = statusChangeMessage.getNodeConnectionStatus(); final String summary = summarizeStatusChange(oldStatus, status); if (!StringUtils.isEmpty(summary)) { addNodeEvent(nodeId, summary); } // Update our counter so that we are in-sync with the cluster on the // most up-to-date version of the NodeConnectionStatus' Update Identifier. // We do this so that we can accurately compare status updates that are generated // locally against those generated from other nodes in the cluster. NodeConnectionStatus.updateIdGenerator(updatedStatus.getUpdateIdentifier()); if (isActiveClusterCoordinator()) { notifyOthersOfNodeStatusChange(statusChangeMessage.getNodeConnectionStatus()); } }
@Override public boolean resetNodeStatus(final NodeConnectionStatus connectionStatus, final long qualifyingUpdateId) { final NodeIdentifier nodeId = connectionStatus.getNodeIdentifier(); final NodeConnectionStatus currentStatus = getConnectionStatus(nodeId); if (currentStatus == null) { return replaceNodeStatus(nodeId, null, connectionStatus); } else if (currentStatus.getUpdateIdentifier() == qualifyingUpdateId) { return replaceNodeStatus(nodeId, currentStatus, connectionStatus); } // The update identifier is not the same. We will not replace the value return false; }
nodeDto.setAddress(nodeId.getApiAddress()); nodeDto.setApiPort(nodeId.getApiPort()); nodeDto.setStatus(status.getState().name()); nodeDto.setRoles(roles); if (status.getConnectionRequestTime() != null) { final Date connectionRequested = new Date(status.getConnectionRequestTime()); nodeDto.setConnectionRequested(connectionRequested);
private NodeIdentifier resolveNodeId(final NodeIdentifier proposedIdentifier) { final NodeConnectionStatus proposedConnectionStatus = new NodeConnectionStatus(proposedIdentifier, DisconnectionCode.NOT_YET_CONNECTED); final NodeConnectionStatus existingStatus = nodeStatuses.putIfAbsent(proposedIdentifier, proposedConnectionStatus); NodeIdentifier resolvedNodeId = proposedIdentifier; if (existingStatus == null) { // there is no node with that ID resolvedNodeId = proposedIdentifier; logger.debug("No existing node with ID {}; resolved node ID is as-proposed", proposedIdentifier.getFullDescription()); onNodeAdded(resolvedNodeId, true); } else if (existingStatus.getNodeIdentifier().logicallyEquals(proposedIdentifier)) { // there is a node with that ID but it's the same node. resolvedNodeId = proposedIdentifier; logger.debug("A node already exists with ID {} and is logically equivalent; resolved node ID is as-proposed: {}", proposedIdentifier.getId(), proposedIdentifier.getFullDescription()); } else { // there is a node with that ID and it's a different node resolvedNodeId = new NodeIdentifier(UUID.randomUUID().toString(), proposedIdentifier.getApiAddress(), proposedIdentifier.getApiPort(), proposedIdentifier.getSocketAddress(), proposedIdentifier.getSocketPort(), proposedIdentifier.getLoadBalanceAddress(), proposedIdentifier.getLoadBalancePort(), proposedIdentifier.getSiteToSiteAddress(), proposedIdentifier.getSiteToSitePort(), proposedIdentifier.getSiteToSiteHttpApiPort(), proposedIdentifier.isSiteToSiteSecure()); logger.debug("A node already exists with ID {}. Proposed Node Identifier was {}; existing Node Identifier is {}; Resolved Node Identifier is {}", proposedIdentifier.getId(), proposedIdentifier.getFullDescription(), getNodeIdentifier(proposedIdentifier.getId()).getFullDescription(), resolvedNodeId.getFullDescription()); } return resolvedNodeId; }
final NodeConnectionStatus existingStatus = this.nodeStatuses.putIfAbsent(connectionStatus.getNodeIdentifier(), connectionStatus); if (existingStatus == null) { onNodeAdded(connectionStatus.getNodeIdentifier(), true); return connectionStatus.getNodeIdentifier(); } else { return existingStatus.getNodeIdentifier();
@Override public Map<NodeConnectionState, List<NodeIdentifier>> getConnectionStates() { final Map<NodeConnectionState, List<NodeIdentifier>> connectionStates = new HashMap<>(); for (final Map.Entry<NodeIdentifier, NodeConnectionStatus> entry : nodeStatuses.entrySet()) { final NodeConnectionState state = entry.getValue().getState(); final List<NodeIdentifier> nodeIds = connectionStates.computeIfAbsent(state, s -> new ArrayList<>()); nodeIds.add(entry.getKey()); } return connectionStates; }
@Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (!(obj instanceof NodeConnectionStatus)) { return false; } NodeConnectionStatus other = (NodeConnectionStatus) obj; return Objects.deepEquals(getNodeIdentifier(), other.getNodeIdentifier()) && Objects.deepEquals(getState(), other.getState()); } }
final List<NodeConnectionStatus> nodeStatusList = payload.getClusterStatus(); final Map<NodeIdentifier, Long> updateIdMap = nodeStatusList.stream().collect( Collectors.toMap(status -> status.getNodeIdentifier(), status -> status.getUpdateIdentifier())); final NodeIdentifier nodeId = updatedStatus.getNodeIdentifier(); final Long updateId = updateIdMap.get(nodeId); logger.info("After receiving heartbeat response, updated status of {} to {}", updatedStatus.getNodeIdentifier(), updatedStatus); } else { logger.debug("After receiving heartbeat response, did not update status of {} to {} because the update is out-of-date", updatedStatus.getNodeIdentifier(), updatedStatus);
/** * Notifies other nodes that the status of a node changed * * @param updatedStatus the updated status for a node in the cluster * @param notifyAllNodes if <code>true</code> will notify all nodes. If * <code>false</code>, will notify only the cluster coordinator */ void notifyOthersOfNodeStatusChange(final NodeConnectionStatus updatedStatus, final boolean notifyAllNodes, final boolean waitForCoordinator) { // If this node is the active cluster coordinator, then we are going to replicate to all nodes. // Otherwise, get the active coordinator (or wait for one to become active) and then notify the coordinator. final Set<NodeIdentifier> nodesToNotify; if (notifyAllNodes) { nodesToNotify = getNodeIdentifiers(); // Do not notify ourselves because we already know about the status update. nodesToNotify.remove(getLocalNodeIdentifier()); } else if (waitForCoordinator) { nodesToNotify = Collections.singleton(waitForElectedClusterCoordinator()); } else { final NodeIdentifier nodeId = getElectedActiveCoordinatorNode(); if (nodeId == null) { return; } nodesToNotify = Collections.singleton(nodeId); } final NodeStatusChangeMessage message = new NodeStatusChangeMessage(); message.setNodeId(updatedStatus.getNodeIdentifier()); message.setNodeConnectionStatus(updatedStatus); senderListener.notifyNodeStatusChange(nodesToNotify, message); }
@Override public AdaptedNodeConnectionStatus marshal(final NodeConnectionStatus toAdapt) throws Exception { final AdaptedNodeConnectionStatus adapted = new AdaptedNodeConnectionStatus(); if (toAdapt != null) { adapted.setUpdateId(toAdapt.getUpdateIdentifier()); adapted.setNodeId(toAdapt.getNodeIdentifier()); adapted.setConnectionRequestTime(toAdapt.getConnectionRequestTime()); adapted.setOffloadCode(toAdapt.getOffloadCode()); adapted.setDisconnectCode(toAdapt.getDisconnectCode()); adapted.setReason(toAdapt.getReason()); adapted.setState(toAdapt.getState()); } return adapted; } }
@Override public String toString() { final StringBuilder sb = new StringBuilder(); final NodeConnectionState state = getState(); sb.append("NodeConnectionStatus[nodeId=").append(nodeId).append(", state=").append(state); if (state == NodeConnectionState.OFFLOADED || state == NodeConnectionState.OFFLOADING) { sb.append(", Offload Code=").append(getOffloadCode()).append(", Offload Reason=").append(getReason()); } if (state == NodeConnectionState.DISCONNECTED || state == NodeConnectionState.DISCONNECTING) { sb.append(", Disconnect Code=").append(getDisconnectCode()).append(", Disconnect Reason=").append(getReason()); } sb.append(", updateId=").append(getUpdateIdentifier()); sb.append("]"); return sb.toString(); }