public static RestoreInProgress.Entry restoreInProgress(ClusterState state, String restoreUUID) { final RestoreInProgress restoreInProgress = state.custom(RestoreInProgress.TYPE); if (restoreInProgress != null) { return restoreInProgress.get(restoreUUID); } return null; }
/** * Checks if a repository is currently in use by one of the snapshots * * @param clusterState cluster state * @param repository repository id * @return true if repository is currently in use by one of the running snapshots */ public static boolean isRepositoryInUse(ClusterState clusterState, String repository) { RestoreInProgress restoreInProgress = clusterState.custom(RestoreInProgress.TYPE); if (restoreInProgress != null) { for (RestoreInProgress.Entry entry: restoreInProgress) { if (repository.equals(entry.snapshot().getRepository())) { return true; } } } return false; }
/** * Checks if a repository is currently in use by one of the snapshots * * @param clusterState cluster state * @param repository repository id * @return true if repository is currently in use by one of the running snapshots */ public static boolean isRepositoryInUse(ClusterState clusterState, String repository) { SnapshotsInProgress snapshots = clusterState.custom(SnapshotsInProgress.TYPE); if (snapshots != null) { for (SnapshotsInProgress.Entry snapshot : snapshots.entries()) { if (repository.equals(snapshot.snapshot().getRepository())) { return true; } } } SnapshotDeletionsInProgress deletionsInProgress = clusterState.custom(SnapshotDeletionsInProgress.TYPE); if (deletionsInProgress != null) { for (SnapshotDeletionsInProgress.Entry entry : deletionsInProgress.getEntries()) { if (entry.getSnapshot().getRepository().equals(repository)) { return true; } } } return false; }
@Override public ClusterState execute(ClusterState currentState) { SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE); if (snapshots != null) { boolean changed = false; ArrayList<SnapshotsInProgress.Entry> entries = new ArrayList<>(); for (SnapshotsInProgress.Entry entry : snapshots.entries()) { if (entry.snapshot().equals(snapshot)) { changed = true; } else { entries.add(entry); } } if (changed) { snapshots = new SnapshotsInProgress(entries.toArray(new SnapshotsInProgress.Entry[entries.size()])); return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, snapshots).build(); } } return currentState; }
/** * Check if any of the indices to be closed are currently being restored from a snapshot and fail closing if such an index * is found as closing an index that is being restored makes the index unusable (it cannot be recovered). */ public static void checkIndexClosing(ClusterState currentState, Set<IndexMetaData> indices) { RestoreInProgress restore = currentState.custom(RestoreInProgress.TYPE); if (restore != null) { Set<Index> indicesToFail = null; for (RestoreInProgress.Entry entry : restore) { for (ObjectObjectCursor<ShardId, RestoreInProgress.ShardRestoreStatus> shard : entry.shards()) { if (!shard.value.state().completed()) { IndexMetaData indexMetaData = currentState.metaData().index(shard.key.getIndex()); if (indexMetaData != null && indices.contains(indexMetaData)) { if (indicesToFail == null) { indicesToFail = new HashSet<>(); } indicesToFail.add(shard.key.getIndex()); } } } } if (indicesToFail != null) { throw new IllegalArgumentException("Cannot close indices that are being restored: " + indicesToFail); } } }
@Override public ClusterState execute(ClusterState currentState) { SnapshotDeletionsInProgress deletions = currentState.custom(SnapshotDeletionsInProgress.TYPE); if (deletions != null) { boolean changed = false; if (deletions.hasDeletionsInProgress()) { assert deletions.getEntries().size() == 1 : "should have exactly one deletion in progress"; SnapshotDeletionsInProgress.Entry entry = deletions.getEntries().get(0); deletions = deletions.withRemovedEntry(entry); changed = true; } if (changed) { return ClusterState.builder(currentState).putCustom(SnapshotDeletionsInProgress.TYPE, deletions).build(); } } return currentState; }
@Override public void clusterChanged(ClusterChangedEvent event) { try { SnapshotsInProgress previousSnapshots = event.previousState().custom(SnapshotsInProgress.TYPE); SnapshotsInProgress currentSnapshots = event.state().custom(SnapshotsInProgress.TYPE); if ((previousSnapshots == null && currentSnapshots != null) || (previousSnapshots != null && previousSnapshots.equals(currentSnapshots) == false)) { processIndexShardSnapshots(event); } String previousMasterNodeId = event.previousState().nodes().getMasterNodeId(); String currentMasterNodeId = event.state().nodes().getMasterNodeId(); if (currentMasterNodeId != null && currentMasterNodeId.equals(previousMasterNodeId) == false) { syncShardStatsOnNewMaster(event); } } catch (Exception e) { logger.warn("Failed to update snapshot state ", e); } }
SnapshotsInProgress snapshotsInProgress = clusterService.state().custom(SnapshotsInProgress.TYPE); if (snapshotsInProgress == null || snapshotsInProgress.entries().isEmpty()) { return Collections.emptyList();
@Override public ClusterTasksResult<Task> execute(final ClusterState currentState, final List<Task> tasks) throws Exception { final ClusterTasksResult.Builder<Task> resultBuilder = ClusterTasksResult.<Task>builder().successes(tasks); Set<String> completedRestores = tasks.stream().map(e -> e.uuid).collect(Collectors.toSet()); RestoreInProgress.Builder restoreInProgressBuilder = new RestoreInProgress.Builder(); final RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE); boolean changed = false; if (restoreInProgress != null) { for (RestoreInProgress.Entry entry : restoreInProgress) { if (completedRestores.contains(entry.uuid())) { changed = true; } else { restoreInProgressBuilder.add(entry); } } } if (changed == false) { return resultBuilder.build(currentState); } ImmutableOpenMap.Builder<String, ClusterState.Custom> builder = ImmutableOpenMap.builder(currentState.getCustoms()); builder.put(RestoreInProgress.TYPE, restoreInProgressBuilder.build()); ImmutableOpenMap<String, ClusterState.Custom> customs = builder.build(); return resultBuilder.build(ClusterState.builder(currentState).customs(customs).build()); }
private static Set<Index> indicesToFailForCloseOrDeletion(ClusterState currentState, Set<IndexMetaData> indices) { SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE); Set<Index> indicesToFail = null; if (snapshots != null) {
@Override public ClusterState execute(ClusterState currentState) { validate(request, currentState); SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE); if (deletionsInProgress != null && deletionsInProgress.hasDeletionsInProgress()) { throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName, "cannot snapshot while a snapshot deletion is in-progress"); } SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE); if (snapshots == null || snapshots.entries().isEmpty()) { // Store newSnapshot here to be processed in clusterStateProcessed List<String> indices = Arrays.asList(indexNameExpressionResolver.concreteIndexNames(currentState, request.indicesOptions(), request.indices())); logger.trace("[{}][{}] creating snapshot for indices [{}]", repositoryName, snapshotName, indices); List<IndexId> snapshotIndices = repositoryData.resolveNewIndices(indices); newSnapshot = new SnapshotsInProgress.Entry(new Snapshot(repositoryName, snapshotId), request.includeGlobalState(), request.partial(), State.INIT, snapshotIndices, System.currentTimeMillis(), repositoryData.getGenId(), null); snapshots = new SnapshotsInProgress(newSnapshot); } else { throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName, " a snapshot is already running"); } return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, snapshots).build(); }
/** * Finalizes a snapshot deletion in progress if the current node is the master but it * was not master in the previous cluster state and there is still a lingering snapshot * deletion in progress in the cluster state. This means that the old master failed * before it could clean up an in-progress snapshot deletion. We attempt to delete the * snapshot files and remove the deletion from the cluster state. It is possible that the * old master was in a state of long GC and then it resumes and tries to delete the snapshot * that has already been deleted by the current master. This is acceptable however, since * the old master's snapshot deletion will just respond with an error but in actuality, the * snapshot was deleted and a call to GET snapshots would reveal that the snapshot no longer exists. */ private void finalizeSnapshotDeletionFromPreviousMaster(ClusterChangedEvent event) { if (event.localNodeMaster() && event.previousState().nodes().isLocalNodeElectedMaster() == false) { SnapshotDeletionsInProgress deletionsInProgress = event.state().custom(SnapshotDeletionsInProgress.TYPE); if (deletionsInProgress != null && deletionsInProgress.hasDeletionsInProgress()) { assert deletionsInProgress.getEntries().size() == 1 : "only one in-progress deletion allowed per cluster"; SnapshotDeletionsInProgress.Entry entry = deletionsInProgress.getEntries().get(0); deleteSnapshotFromRepository(entry.getSnapshot(), null, entry.getRepositoryStateId()); } } }
@Override public ClusterState execute(ClusterState currentState) throws Exception { RoutingTable routingTable = currentState.routingTable(); SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE); if (snapshots != null) { boolean changed = false; ArrayList<SnapshotsInProgress.Entry> entries = new ArrayList<>(); for (final SnapshotsInProgress.Entry snapshot : snapshots.entries()) { SnapshotsInProgress.Entry updatedSnapshot = snapshot; if (snapshot.state() == State.STARTED) { ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards = processWaitingShards(snapshot.shards(), routingTable); if (shards != null) { changed = true; if (!snapshot.state().completed() && completed(shards.values())) { updatedSnapshot = new SnapshotsInProgress.Entry(snapshot, State.SUCCESS, shards); endSnapshot(updatedSnapshot); } else { updatedSnapshot = new SnapshotsInProgress.Entry(snapshot, shards); } } entries.add(updatedSnapshot); } } if (changed) { snapshots = new SnapshotsInProgress(entries.toArray(new SnapshotsInProgress.Entry[entries.size()])); return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, snapshots).build(); } } return currentState; }
/** * Removes a finished snapshot from the cluster state. This can happen if the previous * master node processed a cluster state update that marked the snapshot as finished, * but the previous master node died before removing the snapshot in progress from the * cluster state. It is then the responsibility of the new master node to end the * snapshot and remove it from the cluster state. */ private void removeFinishedSnapshotFromClusterState(ClusterChangedEvent event) { if (event.localNodeMaster() && !event.previousState().nodes().isLocalNodeElectedMaster()) { SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE); if (snapshotsInProgress != null && !snapshotsInProgress.entries().isEmpty()) { for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) { if (entry.state().completed()) { endSnapshot(entry); } } } } }
private void cleanupRestoreState(ClusterChangedEvent event) { ClusterState state = event.state(); RestoreInProgress restoreInProgress = state.custom(RestoreInProgress.TYPE); if (restoreInProgress != null) { for (RestoreInProgress.Entry entry : restoreInProgress) { if (entry.state().completed()) { assert completed(entry.shards()) : "state says completed but restore entries are not"; clusterService.submitStateUpdateTask( "clean up snapshot restore state", new CleanRestoreStateTaskExecutor.Task(entry.uuid()), ClusterStateTaskConfig.build(Priority.URGENT), cleanRestoreStateTaskExecutor, cleanRestoreStateTaskExecutor); } } } }
SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE); if (snapshotsInProgress == null) { return;
@Override public ClusterTasksResult<UpdateIndexShardSnapshotStatusRequest> execute(ClusterState currentState, List<UpdateIndexShardSnapshotStatusRequest> tasks) throws Exception { final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE); if (snapshots != null) { int changedCount = 0;
private boolean removedNodesCleanupNeeded(ClusterChangedEvent event) { SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE); if (snapshotsInProgress == null) { return false; } // Check if we just became the master boolean newMaster = !event.previousState().nodes().isLocalNodeElectedMaster(); for (SnapshotsInProgress.Entry snapshot : snapshotsInProgress.entries()) { if (newMaster && (snapshot.state() == State.SUCCESS || snapshot.state() == State.INIT)) { // We just replaced old master and snapshots in intermediate states needs to be cleaned return true; } for (DiscoveryNode node : event.nodesDelta().removedNodes()) { for (ObjectCursor<ShardSnapshotStatus> shardStatus : snapshot.shards().values()) { if (!shardStatus.value.state().completed() && node.getId().equals(shardStatus.value.nodeId())) { // At least one shard was running on the removed node - we need to fail it return true; } } } } return false; }
final RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE); if (restoreInProgress != null) { RestoreInProgress updatedRestoreInProgress = RestoreService.updateRestoreStateWithDeletedIndices(restoreInProgress, indices);
private boolean waitingShardsStartedOrUnassigned(ClusterChangedEvent event) { SnapshotsInProgress curr = event.state().custom(SnapshotsInProgress.TYPE); if (curr != null) { for (SnapshotsInProgress.Entry entry : curr.entries()) { if (entry.state() == State.STARTED && !entry.waitingIndices().isEmpty()) { for (ObjectCursor<String> index : entry.waitingIndices().keys()) { if (event.indexRoutingTableChanged(index.value)) { IndexRoutingTable indexShardRoutingTable = event.state().getRoutingTable().index(index.value); for (ShardId shardId : entry.waitingIndices().get(index.value)) { ShardRouting shardRouting = indexShardRoutingTable.shard(shardId.id()).primaryShard(); if (shardRouting != null && (shardRouting.started() || shardRouting.unassigned())) { return true; } } } } } } } return false; }