public String toString() { return "Context(Distributor " + calculatedDistributor + ", state version " + usedState.getVersion() + ")"; } }
private void resetCachedStateIfClusterStateVersionLikelyRolledBack(ClusterState newState) { if (cachedClusterState != null && cachedClusterState.getVersion() > newState.getVersion()) { if (++oldClusterVersionGottenCount >= maxOldClusterVersionBeforeSendingRandom) { oldClusterVersionGottenCount = 0; cachedClusterState = null; } } }
private void updateCachedRoutingStateFromWrongDistribution(MessageContext context, ClusterState newState) { if (cachedClusterState == null || newState.getVersion() >= cachedClusterState.getVersion()) { cachedClusterState = newState; if (newState.getClusterState().equals(State.UP)) { hostFetcher.updateValidTargets(newState); } } else if (newState.getVersion() + 2000000000 < cachedClusterState.getVersion()) { cachedClusterState = null; } else if (context.calculatedDistributor != null) { persistentFailureChecker.addFailure(context.calculatedDistributor); } }
private void traceReplyFromRandomDistributor(WrongDistributionReply reply, ClusterState newState) { if (!reply.getTrace().shouldTrace(1)) { return; } if (cachedClusterState == null) { reply.getTrace().trace(1, "Message sent to * with no previous state, received version " + newState.getVersion()); } else if (newState.getVersion() == cachedClusterState.getVersion()) { reply.getTrace().trace(1, "Message sent to * found that cluster state version " + newState.getVersion() + " was correct."); } else if (newState.getVersion() > cachedClusterState.getVersion()) { reply.getTrace().trace(1, "Message sent to * updated cluster state to version " + newState.getVersion()); } else { reply.getTrace().trace(1, "Message sent to * retrieved older cluster state version " + newState.getVersion()); } }
private void markReplyAsImmediateRetryIfNewStateObserved(WrongDistributionReply reply, MessageContext context, ClusterState newState) { if (context.usedState != null && newState.getVersion() <= context.usedState.getVersion()) { if (reply.getRetryDelay() <= 0.0) { reply.setRetryDelay(-1); } } else { if (reply.getRetryDelay() <= 0.0) { reply.setRetryDelay(0); } } }
private void traceReplyFromSpecificDistributor(WrongDistributionReply reply, MessageContext context, ClusterState newState) { if (context.usedState == null) { String msg = "Used state must be set as distributor is calculated. Bug."; reply.getTrace().trace(1, msg); log.log(LogLevel.ERROR, msg); } else if (newState.getVersion() == context.usedState.getVersion()) { String msg = "Message sent to distributor " + context.calculatedDistributor + " retrieved cluster state version " + newState.getVersion() + " which was the state we used to calculate distributor as target last time."; reply.getTrace().trace(1, msg); // Client load can be rejected towards distributors even with a matching cluster state version. // This usually happens during a node fail-over transition, where the target distributor will // reject an operation bound to a particular bucket if it does not own the bucket in _both_ // the current and the next (transition target) state. Since it can happen during normal operation // and will happen per client operation, we keep this as debug level to prevent spamming the logs. log.log(LogLevel.DEBUG, msg); } else if (newState.getVersion() > context.usedState.getVersion()) { if (reply.getTrace().shouldTrace(1)) { reply.getTrace().trace(1, "Message sent to distributor " + context.calculatedDistributor + " updated cluster state from version " + context.usedState.getVersion() + " to " + newState.getVersion()); } } else { if (reply.getTrace().shouldTrace(1)) { reply.getTrace().trace(1, "Message sent to distributor " + context.calculatedDistributor + " returned older cluster state version " + newState.getVersion()); } } }
if (context.shouldTrace(1)) { context.trace(1, "Using distributor " + messageContext.calculatedDistributor + " for " + bucketId + " as our state version is " + cachedClusterState.getVersion());
throw new NoDistributorsAvailableException("No distributors available in cluster state version " + state.getVersion());