private void performOnReplicas(final ReplicaRequest replicaRequest, final long globalCheckpoint, final long maxSeqNoOfUpdatesOrDeletes, final ReplicationGroup replicationGroup) { // for total stats, add number of unassigned shards and // number of initializing shards that are not ready yet to receive operations (recovery has not opened engine yet on the target) totalShards.addAndGet(replicationGroup.getSkippedShards().size()); final ShardRouting primaryRouting = primary.routingEntry(); for (final ShardRouting shard : replicationGroup.getReplicationTargets()) { if (shard.isSameAllocation(primaryRouting) == false) { performOnReplica(shard, replicaRequest, globalCheckpoint, maxSeqNoOfUpdatesOrDeletes); } } }
/** * Removes shard entries from the failed shards cache that are no longer allocated to this node by the master. * Sends shard failures for shards that are marked as actively allocated to this node but don't actually exist on the node. * Resends shard failures for shards that are still marked as allocated to this node but previously failed. * * @param state new cluster state */ private void updateFailedShardsCache(final ClusterState state) { RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId()); if (localRoutingNode == null) { failedShardsCache.clear(); return; } DiscoveryNode masterNode = state.nodes().getMasterNode(); // remove items from cache which are not in our routing table anymore and resend failures that have not executed on master yet for (Iterator<Map.Entry<ShardId, ShardRouting>> iterator = failedShardsCache.entrySet().iterator(); iterator.hasNext(); ) { ShardRouting failedShardRouting = iterator.next().getValue(); ShardRouting matchedRouting = localRoutingNode.getByShardId(failedShardRouting.shardId()); if (matchedRouting == null || matchedRouting.isSameAllocation(failedShardRouting) == false) { iterator.remove(); } else { if (masterNode != null) { // TODO: can we remove this? Is resending shard failures the responsibility of shardStateAction? String message = "master " + masterNode + " has not removed previously failed shard. resending shard failure"; logger.trace("[{}] re-sending failed shard [{}], reason [{}]", matchedRouting.shardId(), matchedRouting, message); shardStateAction.localShardFailed(matchedRouting, message, null, SHARD_STATE_ACTION_LISTENER, state); } } } }
private Decision decideSameNode(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation, Iterable<ShardRouting> assignedShards) { for (ShardRouting assignedShard : assignedShards) { if (node.nodeId().equals(assignedShard.currentNodeId())) { if (assignedShard.isSameAllocation(shardRouting)) { return allocation.decision(Decision.NO, NAME, "the shard cannot be allocated to the node on which it already exists [%s]", shardRouting.toString()); } else { return allocation.decision(Decision.NO, NAME, "the shard cannot be allocated to the same node on which a copy of the shard already exists [%s]", assignedShard.toString()); } } } return allocation.decision(Decision.YES, NAME, "the shard does not exist on the same node"); } }
private void updateShard(DiscoveryNodes nodes, ShardRouting shardRouting, Shard shard, RoutingTable routingTable, ClusterState clusterState) { final ShardRouting currentRoutingEntry = shard.routingEntry(); assert currentRoutingEntry.isSameAllocation(shardRouting) : "local shard has a different allocation id but wasn't cleaning by removeShards. " + "cluster state: " + shardRouting + " local: " + currentRoutingEntry;
private static void persistMetadata( final ShardPath shardPath, final IndexSettings indexSettings, final ShardRouting newRouting, final @Nullable ShardRouting currentRouting, final Logger logger) throws IOException { assert newRouting != null : "newRouting must not be null"; // only persist metadata if routing information that is persisted in shard state metadata actually changed final ShardId shardId = newRouting.shardId(); if (currentRouting == null || currentRouting.primary() != newRouting.primary() || currentRouting.allocationId().equals(newRouting.allocationId()) == false) { assert currentRouting == null || currentRouting.isSameAllocation(newRouting); final String writeReason; if (currentRouting == null) { writeReason = "initial state with allocation id [" + newRouting.allocationId() + "]"; } else { writeReason = "routing changed from " + currentRouting + " to " + newRouting; } logger.trace("{} writing shard state, reason [{}]", shardId, writeReason); final ShardStateMetaData newShardStateMetadata = new ShardStateMetaData(newRouting.primary(), indexSettings.getUUID(), newRouting.allocationId()); ShardStateMetaData.FORMAT.write(newShardStateMetadata, shardPath.getShardStatePath()); } else { logger.trace("{} skip writing shard state, has been written before", shardId); } }
} else if (newShardRouting.isSameAllocation(currentRoutingEntry) == false) { logger.debug("{} removing shard (stale allocation id, stale {}, new {})", shardId, currentRoutingEntry, newShardRouting);
newRouting.shardId() + " on a shard with shardId " + shardId()); if ((currentRouting == null || newRouting.isSameAllocation(currentRouting)) == false) { throw new IllegalArgumentException("Trying to set a routing entry with a different allocation. Current " + currentRouting + ", new " + newRouting);
private void performOnReplicas(final ReplicaRequest replicaRequest, final long globalCheckpoint, final long maxSeqNoOfUpdatesOrDeletes, final ReplicationGroup replicationGroup) { // for total stats, add number of unassigned shards and // number of initializing shards that are not ready yet to receive operations (recovery has not opened engine yet on the target) totalShards.addAndGet(replicationGroup.getSkippedShards().size()); final ShardRouting primaryRouting = primary.routingEntry(); for (final ShardRouting shard : replicationGroup.getReplicationTargets()) { if (shard.isSameAllocation(primaryRouting) == false) { performOnReplica(shard, replicaRequest, globalCheckpoint, maxSeqNoOfUpdatesOrDeletes); } } }
private void performOnReplicas(final ReplicaRequest replicaRequest, final long globalCheckpoint, final ReplicationGroup replicationGroup) { // for total stats, add number of unassigned shards and // number of initializing shards that are not ready yet to receive operations (recovery has not opened engine yet on the target) totalShards.addAndGet(replicationGroup.getSkippedShards().size()); final ShardRouting primaryRouting = primary.routingEntry(); for (final ShardRouting shard : replicationGroup.getReplicationTargets()) { if (shard.isSameAllocation(primaryRouting) == false) { performOnReplica(shard, replicaRequest, globalCheckpoint); } } }
/** * Removes shard entries from the failed shards cache that are no longer allocated to this node by the master. * Sends shard failures for shards that are marked as actively allocated to this node but don't actually exist on the node. * Resends shard failures for shards that are still marked as allocated to this node but previously failed. * * @param state new cluster state */ private void updateFailedShardsCache(final ClusterState state) { RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId()); if (localRoutingNode == null) { failedShardsCache.clear(); return; } DiscoveryNode masterNode = state.nodes().getMasterNode(); // remove items from cache which are not in our routing table anymore and resend failures that have not executed on master yet for (Iterator<Map.Entry<ShardId, ShardRouting>> iterator = failedShardsCache.entrySet().iterator(); iterator.hasNext(); ) { ShardRouting failedShardRouting = iterator.next().getValue(); ShardRouting matchedRouting = localRoutingNode.getByShardId(failedShardRouting.shardId()); if (matchedRouting == null || matchedRouting.isSameAllocation(failedShardRouting) == false) { iterator.remove(); } else { if (masterNode != null) { // TODO: can we remove this? Is resending shard failures the responsibility of shardStateAction? String message = "master " + masterNode + " has not removed previously failed shard. resending shard failure"; logger.trace("[{}] re-sending failed shard [{}], reason [{}]", matchedRouting.shardId(), matchedRouting, message); shardStateAction.localShardFailed(matchedRouting, message, null, SHARD_STATE_ACTION_LISTENER, state); } } } }
/** * Removes shard entries from the failed shards cache that are no longer allocated to this node by the master. * Sends shard failures for shards that are marked as actively allocated to this node but don't actually exist on the node. * Resends shard failures for shards that are still marked as allocated to this node but previously failed. * * @param state new cluster state */ private void updateFailedShardsCache(final ClusterState state) { RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId()); if (localRoutingNode == null) { failedShardsCache.clear(); return; } DiscoveryNode masterNode = state.nodes().getMasterNode(); // remove items from cache which are not in our routing table anymore and resend failures that have not executed on master yet for (Iterator<Map.Entry<ShardId, ShardRouting>> iterator = failedShardsCache.entrySet().iterator(); iterator.hasNext(); ) { ShardRouting failedShardRouting = iterator.next().getValue(); ShardRouting matchedRouting = localRoutingNode.getByShardId(failedShardRouting.shardId()); if (matchedRouting == null || matchedRouting.isSameAllocation(failedShardRouting) == false) { iterator.remove(); } else { if (masterNode != null) { // TODO: can we remove this? Is resending shard failures the responsibility of shardStateAction? String message = "master " + masterNode + " has not removed previously failed shard. resending shard failure"; logger.trace("[{}] re-sending failed shard [{}], reason [{}]", matchedRouting.shardId(), matchedRouting, message); shardStateAction.localShardFailed(matchedRouting, message, null, SHARD_STATE_ACTION_LISTENER, state); } } } }
private Decision decideSameNode(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation, Iterable<ShardRouting> assignedShards) { for (ShardRouting assignedShard : assignedShards) { if (node.nodeId().equals(assignedShard.currentNodeId())) { if (assignedShard.isSameAllocation(shardRouting)) { return allocation.decision(Decision.NO, NAME, "the shard cannot be allocated to the node on which it already exists [%s]", shardRouting.toString()); } else { return allocation.decision(Decision.NO, NAME, "the shard cannot be allocated to the same node on which a copy of the shard already exists [%s]", assignedShard.toString()); } } } return allocation.decision(Decision.YES, NAME, "the shard does not exist on the same node"); } }
private Decision decideSameNode(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation, Iterable<ShardRouting> assignedShards) { for (ShardRouting assignedShard : assignedShards) { if (node.nodeId().equals(assignedShard.currentNodeId())) { if (assignedShard.isSameAllocation(shardRouting)) { return allocation.decision(Decision.NO, NAME, "the shard cannot be allocated to the node on which it already exists [%s]", shardRouting.toString()); } else { return allocation.decision(Decision.NO, NAME, "the shard cannot be allocated to the same node on which a copy of the shard already exists [%s]", assignedShard.toString()); } } } return allocation.decision(Decision.YES, NAME, "the shard does not exist on the same node"); } }
private Decision decideSameNode(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation, Iterable<ShardRouting> assignedShards) { for (ShardRouting assignedShard : assignedShards) { if (node.nodeId().equals(assignedShard.currentNodeId())) { if (assignedShard.isSameAllocation(shardRouting)) { return allocation.decision(Decision.NO, NAME, "the shard cannot be allocated to the node on which it already exists [%s]", shardRouting.toString()); } else { return allocation.decision(Decision.NO, NAME, "the shard cannot be allocated to the same node on which a copy of the shard already exists [%s]", assignedShard.toString()); } } } return allocation.decision(Decision.YES, NAME, "the shard does not exist on the same node"); } }
private void updateShard(DiscoveryNodes nodes, ShardRouting shardRouting, Shard shard, RoutingTable routingTable, ClusterState clusterState) { final ShardRouting currentRoutingEntry = shard.routingEntry(); assert currentRoutingEntry.isSameAllocation(shardRouting) : "local shard has a different allocation id but wasn't cleaning by removeShards. " + "cluster state: " + shardRouting + " local: " + currentRoutingEntry; try { shard.moveToStart(); } catch (Exception e) { failAndRemoveShard(shardRouting, true, "failed updating shard routing entry", e, clusterState); return; } final IndexShardState state = shard.state(); if (shardRouting.initializing() && (state == IndexShardState.STARTED || state == IndexShardState.POST_RECOVERY)) { // the master thinks we are initializing, but we are already started or on POST_RECOVERY and waiting // for master to confirm a shard started message (either master failover, or a cluster event before // we managed to tell the master we started), mark us as started if (logger.isTraceEnabled()) { logger.trace("{} master marked shard as initializing, but shard has state [{}], resending shard started to {}", shardRouting.shardId(), state, nodes.getMasterNode()); } } }
private static void persistMetadata( final ShardPath shardPath, final IndexSettings indexSettings, final ShardRouting newRouting, final @Nullable ShardRouting currentRouting, final Logger logger) throws IOException { assert newRouting != null : "newRouting must not be null"; // only persist metadata if routing information that is persisted in shard state metadata actually changed final ShardId shardId = newRouting.shardId(); if (currentRouting == null || currentRouting.primary() != newRouting.primary() || currentRouting.allocationId().equals(newRouting.allocationId()) == false) { assert currentRouting == null || currentRouting.isSameAllocation(newRouting); final String writeReason; if (currentRouting == null) { writeReason = "initial state with allocation id [" + newRouting.allocationId() + "]"; } else { writeReason = "routing changed from " + currentRouting + " to " + newRouting; } logger.trace("{} writing shard state, reason [{}]", shardId, writeReason); final ShardStateMetaData newShardStateMetadata = new ShardStateMetaData(newRouting.primary(), indexSettings.getUUID(), newRouting.allocationId()); ShardStateMetaData.FORMAT.write(newShardStateMetadata, shardPath.getShardStatePath()); } else { logger.trace("{} skip writing shard state, has been written before", shardId); } }
private void updateShard(DiscoveryNodes nodes, ShardRouting shardRouting, Shard shard, RoutingTable routingTable, ClusterState clusterState) { final ShardRouting currentRoutingEntry = shard.routingEntry(); assert currentRoutingEntry.isSameAllocation(shardRouting) : "local shard has a different allocation id but wasn't cleaning by removeShards. " + "cluster state: " + shardRouting + " local: " + currentRoutingEntry;
} else if (newShardRouting.isSameAllocation(currentRoutingEntry) == false) { logger.debug("{} removing shard (stale allocation id, stale {}, new {})", shardId, currentRoutingEntry, newShardRouting);
} else if (newShardRouting.isSameAllocation(currentRoutingEntry) == false) { logger.debug("{} removing shard (stale allocation id, stale {}, new {})", shardId, currentRoutingEntry, newShardRouting);
} else if (newShardRouting.isSameAllocation(currentRoutingEntry) == false) { logger.debug("{} removing shard (stale allocation id, stale {}, new {})", shardId, currentRoutingEntry, newShardRouting);