/** * This method returns true if the current node is allowed to delete the given index. * This is the case if the index is deleted in the metadata or there is no allocation * on the local node and the index isn't on a shared file system. * @param index {@code Index} to check whether deletion is allowed * @param indexSettings {@code IndexSettings} for the given index * @return true if the index can be deleted on this node */ public boolean canDeleteIndexContents(Index index, IndexSettings indexSettings) { // index contents can be deleted if its an already closed index (so all its resources have // already been relinquished) final IndexService indexService = indexService(index); if (indexService == null && nodeEnv.hasNodeFile()) { return true; } return false; }
/** * Process dangling indices based on the provided meta data, handling cleanup, finding * new dangling indices, and allocating outstanding ones. */ public void processDanglingIndices(final MetaData metaData) { if (nodeEnv.hasNodeFile() == false) { return; } cleanupAllocatedDangledIndices(metaData); findNewAndAddDanglingIndices(metaData); allocateDanglingIndices(); }
public FsInfo stats(FsInfo previous, @Nullable ClusterInfo clusterInfo) throws IOException { if (!nodeEnv.hasNodeFile()) { return new FsInfo(System.currentTimeMillis(), null, new FsInfo.Path[0]); } NodePath[] dataLocations = nodeEnv.nodePaths(); FsInfo.Path[] paths = new FsInfo.Path[dataLocations.length]; for (int i = 0; i < dataLocations.length; i++) { paths[i] = getFSInfo(dataLocations[i]); } FsInfo.IoStats ioStats = null; if (Constants.LINUX) { Set<Tuple<Integer, Integer>> devicesNumbers = new HashSet<>(); for (int i = 0; i < dataLocations.length; i++) { if (dataLocations[i].majorDeviceNumber != -1 && dataLocations[i].minorDeviceNumber != -1) { devicesNumbers.add(Tuple.tuple(dataLocations[i].majorDeviceNumber, dataLocations[i].minorDeviceNumber)); } } ioStats = ioStats(devicesNumbers, previous); } DiskUsage leastDiskEstimate = null; DiskUsage mostDiskEstimate = null; if (clusterInfo != null) { leastDiskEstimate = clusterInfo.getNodeLeastAvailableDiskUsages().get(nodeEnv.nodeId()); mostDiskEstimate = clusterInfo.getNodeMostAvailableDiskUsages().get(nodeEnv.nodeId()); } return new FsInfo(System.currentTimeMillis(), ioStats, paths, leastDiskEstimate, mostDiskEstimate); }
/** * Deletes an index that is not assigned to this node. This method cleans up all disk folders relating to the index * but does not deal with in-memory structures. For those call {@link #removeIndex(Index, IndexRemovalReason, String)} */ @Override public void deleteUnassignedIndex(String reason, IndexMetaData metaData, ClusterState clusterState) { if (nodeEnv.hasNodeFile()) { String indexName = metaData.getIndex().getName(); try { if (clusterState.metaData().hasIndex(indexName)) { final IndexMetaData index = clusterState.metaData().index(indexName); throw new IllegalStateException("Can't delete unassigned index store for [" + indexName + "] - it's still part of " + "the cluster state [" + index.getIndexUUID() + "] [" + metaData.getIndexUUID() + "]"); } deleteIndexStore(reason, metaData, clusterState); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("[{}] failed to delete unassigned index (reason [{}])", metaData.getIndex(), reason), e); } } }
throw new IllegalStateException("Cannot delete index [" + index + "], it is still part of the cluster state."); if (nodeEnv.hasNodeFile() && FileSystemUtils.exists(nodeEnv.indexPaths(index))) { final IndexMetaData metaData; try {
/** * Returns <code>ShardDeletionCheckResult</code> signaling whether the shards content for the given shard can be deleted. * * @param shardId the shard to delete. * @param indexSettings the shards's relevant {@link IndexSettings}. This is required to access the indexes settings etc. */ public ShardDeletionCheckResult canDeleteShardContent(ShardId shardId, IndexSettings indexSettings) { assert shardId.getIndex().equals(indexSettings.getIndex()); final IndexService indexService = indexService(shardId.getIndex()); if (nodeEnv.hasNodeFile()) { final boolean isAllocated = indexService != null && indexService.hasShard(shardId.id()); if (isAllocated) { return ShardDeletionCheckResult.STILL_ALLOCATED; // we are allocated - can't delete the shard } else if (indexSettings.hasCustomDataPath()) { // lets see if it's on a custom path (return false if the shared doesn't exist) // we don't need to delete anything that is not there return Files.exists(nodeEnv.resolveCustomLocation(indexSettings, shardId)) ? ShardDeletionCheckResult.FOLDER_FOUND_CAN_DELETE : ShardDeletionCheckResult.NO_FOLDER_FOUND; } else { // lets see if it's path is available (return false if the shared doesn't exist) // we don't need to delete anything that is not there return FileSystemUtils.exists(nodeEnv.availableShardPaths(shardId)) ? ShardDeletionCheckResult.FOLDER_FOUND_CAN_DELETE : ShardDeletionCheckResult.NO_FOLDER_FOUND; } } else { return ShardDeletionCheckResult.NO_LOCAL_STORAGE; } }
/** * Deletes the index store trying to acquire all shards locks for this index. * This method will delete the metadata for the index even if the actual shards can't be locked. * * Package private for testing */ void deleteIndexStore(String reason, IndexMetaData metaData, ClusterState clusterState) throws IOException { if (nodeEnv.hasNodeFile()) { synchronized (this) { Index index = metaData.getIndex(); if (hasIndex(index)) { String localUUid = indexService(index).indexUUID(); throw new IllegalStateException("Can't delete index store for [" + index.getName() + "] - it's still part of the indices service [" + localUUid + "] [" + metaData.getIndexUUID() + "]"); } if (clusterState.metaData().hasIndex(index.getName()) && (clusterState.nodes().getLocalNode().isMasterNode() == true)) { // we do not delete the store if it is a master eligible node and the index is still in the cluster state // because we want to keep the meta data for indices around even if no shards are left here final IndexMetaData idxMeta = clusterState.metaData().index(index.getName()); throw new IllegalStateException("Can't delete index store for [" + index.getName() + "] - it's still part of the " + "cluster state [" + idxMeta.getIndexUUID() + "] [" + metaData.getIndexUUID() + "], " + "we are master eligible, so will keep the index metadata even if no shards are left."); } } final IndexSettings indexSettings = buildIndexSettings(metaData); deleteIndexStore(reason, indexSettings.getIndex(), indexSettings); } }
private void markNodeDataDirsAsPendingForWipe(Node node) { assert Thread.holdsLock(this); NodeEnvironment nodeEnv = node.getNodeEnvironment(); if (nodeEnv.hasNodeFile()) { dataDirToClean.addAll(Arrays.asList(nodeEnv.nodeDataPaths())); } }
/** * This method returns true if the current node is allowed to delete the given index. * This is the case if the index is deleted in the metadata or there is no allocation * on the local node and the index isn't on a shared file system. * @param index {@code Index} to check whether deletion is allowed * @param indexSettings {@code IndexSettings} for the given index * @return true if the index can be deleted on this node */ public boolean canDeleteIndexContents(Index index, IndexSettings indexSettings) { // index contents can be deleted if its an already closed index (so all its resources have // already been relinquished) final IndexService indexService = indexService(index); if (indexService == null && nodeEnv.hasNodeFile()) { return true; } return false; }
private void markNodeDataDirsAsNotEligableForWipe(Node node) { assert Thread.holdsLock(this); NodeEnvironment nodeEnv = node.getNodeEnvironment(); if (nodeEnv.hasNodeFile()) { dataDirToClean.removeAll(Arrays.asList(nodeEnv.nodeDataPaths())); } }
private void clearDataIfNeeded(RestartCallback callback) throws IOException { if (callback.clearData(name)) { NodeEnvironment nodeEnv = node.getNodeEnvironment(); if (nodeEnv.hasNodeFile()) { final Path[] locations = nodeEnv.nodeDataPaths(); logger.debug("removing node data paths: [{}]", Arrays.toString(locations)); IOUtils.rm(locations); } } }
public FsInfo stats() throws IOException { if (!nodeEnv.hasNodeFile()) { return new FsInfo(System.currentTimeMillis(), new FsInfo.Path[0]); } NodePath[] dataLocations = nodeEnv.nodePaths(); FsInfo.Path[] paths = new FsInfo.Path[dataLocations.length]; for (int i = 0; i < dataLocations.length; i++) { paths[i] = getFSInfo(dataLocations[i]); } return new FsInfo(System.currentTimeMillis(), paths); }
/** * Process dangling indices based on the provided meta data, handling cleanup, finding * new dangling indices, and allocating outstanding ones. */ public void processDanglingIndices(final MetaData metaData) { if (nodeEnv.hasNodeFile() == false) { return; } cleanupAllocatedDangledIndices(metaData); findNewAndAddDanglingIndices(metaData); allocateDanglingIndices(); }
/** * Process dangling indices based on the provided meta data, handling cleanup, finding * new dangling indices, and allocating outstanding ones. */ public void processDanglingIndices(final MetaData metaData) { if (nodeEnv.hasNodeFile() == false) { return; } cleanupAllocatedDangledIndices(metaData); findNewAndAddDanglingIndices(metaData); allocateDanglingIndices(); }
/** * Process dangling indices based on the provided meta data, handling cleanup, finding * new dangling indices, and allocating outstanding ones. */ public void processDanglingIndices(MetaData metaData) { if (nodeEnv.hasNodeFile() == false) { return; } cleanupAllocatedDangledIndices(metaData); findNewAndAddDanglingIndices(metaData); allocateDanglingIndices(); }
/** * Process dangling indices based on the provided meta data, handling cleanup, finding * new dangling indices, and allocating outstanding ones. */ public void processDanglingIndices(final MetaData metaData) { if (nodeEnv.hasNodeFile() == false) { return; } cleanupAllocatedDangledIndices(metaData); findNewAndAddDanglingIndices(metaData); allocateDanglingIndices(); }
/** * This method returns true if the current node is allowed to delete the * given index. If the index uses a shared filesystem this method always * returns false. * @param index {@code Index} to check whether deletion is allowed * @param indexSettings {@code Settings} for the given index * @return true if the index can be deleted on this node */ public boolean canDeleteIndexContents(Index index, Settings indexSettings, boolean closed) { final IndexServiceInjectorPair indexServiceInjectorPair = this.indices.get(index.name()); // Closed indices may be deleted, even if they are on a shared // filesystem. Since it is closed we aren't deleting it for relocation if (IndexMetaData.isOnSharedFilesystem(indexSettings) == false || closed) { if (indexServiceInjectorPair == null && nodeEnv.hasNodeFile()) { return true; } } else { logger.trace("{} skipping index directory deletion due to shadow replicas", index); } return false; }
public void deleteClosedIndex(String reason, IndexMetaData metaData, ClusterState clusterState) { if (nodeEnv.hasNodeFile()) { String indexName = metaData.getIndex(); try { if (clusterState.metaData().hasIndex(indexName)) { final IndexMetaData index = clusterState.metaData().index(indexName); throw new IllegalStateException("Can't delete closed index store for [" + indexName + "] - it's still part of the cluster state [" + index.getIndexUUID() + "] [" + metaData.getIndexUUID() + "]"); } deleteIndexStore(reason, metaData, clusterState, true); } catch (IOException e) { logger.warn("[{}] failed to delete closed index", e, metaData.getIndex()); } } }
/** * Deletes an index that is not assigned to this node. This method cleans up all disk folders relating to the index * but does not deal with in-memory structures. For those call {@link #removeIndex(Index, IndexRemovalReason, String)} */ @Override public void deleteUnassignedIndex(String reason, IndexMetaData metaData, ClusterState clusterState) { if (nodeEnv.hasNodeFile()) { String indexName = metaData.getIndex().getName(); try { if (clusterState.metaData().hasIndex(indexName)) { final IndexMetaData index = clusterState.metaData().index(indexName); throw new IllegalStateException("Can't delete unassigned index store for [" + indexName + "] - it's still part of " + "the cluster state [" + index.getIndexUUID() + "] [" + metaData.getIndexUUID() + "]"); } deleteIndexStore(reason, metaData, clusterState); } catch (Exception e) { logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to delete unassigned index (reason [{}])", metaData.getIndex(), reason), e); } } }
/** * Deletes an index that is not assigned to this node. This method cleans up all disk folders relating to the index * but does not deal with in-memory structures. For those call {@link #removeIndex(Index, IndexRemovalReason, String)} */ @Override public void deleteUnassignedIndex(String reason, IndexMetaData metaData, ClusterState clusterState) { if (nodeEnv.hasNodeFile()) { String indexName = metaData.getIndex().getName(); try { if (clusterState.metaData().hasIndex(indexName)) { final IndexMetaData index = clusterState.metaData().index(indexName); throw new IllegalStateException("Can't delete unassigned index store for [" + indexName + "] - it's still part of " + "the cluster state [" + index.getIndexUUID() + "] [" + metaData.getIndexUUID() + "]"); } deleteIndexStore(reason, metaData, clusterState); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("[{}] failed to delete unassigned index (reason [{}])", metaData.getIndex(), reason), e); } } }