locks = null; nodeLockId = -1; nodeMetaData = new NodeMetaData(generateNodeId(settings)); nodeIdConsumer.accept(nodeMetaData.nodeId()); return; this.nodePaths = nodeLock.nodePaths; this.nodeLockId = nodeLock.nodeId; this.nodeMetaData = loadOrCreateNodeMetaData(settings, logger, nodePaths); nodeIdConsumer.accept(nodeMetaData.nodeId()); maybeLogPathDetails(); maybeLogHeapDetails(); applySegmentInfosTrace(settings); assertCanWrite(); success = true; } finally { if (success == false) { close();
public FsInfo stats(FsInfo previous, @Nullable ClusterInfo clusterInfo) throws IOException { if (!nodeEnv.hasNodeFile()) { return new FsInfo(System.currentTimeMillis(), null, new FsInfo.Path[0]); } NodePath[] dataLocations = nodeEnv.nodePaths(); FsInfo.Path[] paths = new FsInfo.Path[dataLocations.length]; for (int i = 0; i < dataLocations.length; i++) { paths[i] = getFSInfo(dataLocations[i]); } FsInfo.IoStats ioStats = null; if (Constants.LINUX) { Set<Tuple<Integer, Integer>> devicesNumbers = new HashSet<>(); for (int i = 0; i < dataLocations.length; i++) { if (dataLocations[i].majorDeviceNumber != -1 && dataLocations[i].minorDeviceNumber != -1) { devicesNumbers.add(Tuple.tuple(dataLocations[i].majorDeviceNumber, dataLocations[i].minorDeviceNumber)); } } ioStats = ioStats(devicesNumbers, previous); } DiskUsage leastDiskEstimate = null; DiskUsage mostDiskEstimate = null; if (clusterInfo != null) { leastDiskEstimate = clusterInfo.getNodeLeastAvailableDiskUsages().get(nodeEnv.nodeId()); mostDiskEstimate = clusterInfo.getNodeMostAvailableDiskUsages().get(nodeEnv.nodeId()); } return new FsInfo(System.currentTimeMillis(), ioStats, paths, leastDiskEstimate, mostDiskEstimate); }
/** * This is a best effort to ensure that we actually have write permissions to write in all our data directories. * This prevents disasters if nodes are started under the wrong username etc. */ private void assertCanWrite() throws IOException { for (Path path : nodeDataPaths()) { // check node-paths are writable tryWriteTempFile(path); } for (String indexFolderName : this.availableIndexFolders()) { for (Path indexPath : this.resolveIndexFolder(indexFolderName)) { // check index paths are writable Path indexStatePath = indexPath.resolve(MetaDataStateFormat.STATE_DIR_NAME); tryWriteTempFile(indexStatePath); tryWriteTempFile(indexPath); try (DirectoryStream<Path> stream = Files.newDirectoryStream(indexPath)) { for (Path shardPath : stream) { String fileName = shardPath.getFileName().toString(); if (Files.isDirectory(shardPath) && fileName.chars().allMatch(Character::isDigit)) { Path indexDir = shardPath.resolve(ShardPath.INDEX_FOLDER_NAME); Path statePath = shardPath.resolve(MetaDataStateFormat.STATE_DIR_NAME); Path translogDir = shardPath.resolve(ShardPath.TRANSLOG_FOLDER_NAME); tryWriteTempFile(indexDir); tryWriteTempFile(translogDir); tryWriteTempFile(statePath); tryWriteTempFile(shardPath); } } } } } }
/** * Deletes a shard data directory iff the shards locks were successfully acquired. * * @param shardId the id of the shard to delete to delete * @throws IOException if an IOException occurs */ public void deleteShardDirectorySafe(ShardId shardId, IndexSettings indexSettings) throws IOException, ShardLockObtainFailedException { final Path[] paths = availableShardPaths(shardId); logger.trace("deleting shard {} directory, paths: [{}]", shardId, paths); try (ShardLock lock = shardLock(shardId)) { deleteShardDirectoryUnderLock(lock, indexSettings); } }
/** * This method walks through the nodes shard paths to find the data and state path for the given shard. If multiple * directories with a valid shard state exist the one with the highest version will be used. * <b>Note:</b> this method resolves custom data locations for the shard. */ public static ShardPath loadShardPath(Logger logger, NodeEnvironment env, ShardId shardId, IndexSettings indexSettings) throws IOException { final Path[] paths = env.availableShardPaths(shardId); final int nodeLockId = env.getNodeLockId(); final Path sharedDataPath = env.sharedDataPath(); return loadShardPath(logger, shardId, indexSettings, paths, nodeLockId, sharedDataPath); }
/** * Returns folder names in ${data.paths}/nodes/{node.id}/indices folder that don't match the given predicate. * @param excludeIndexPathIdsPredicate folder names to exclude */ public Set<String> availableIndexFolders(Predicate<String> excludeIndexPathIdsPredicate) throws IOException { if (nodePaths == null || locks == null) { throw new IllegalStateException("node is not configured to store local location"); } assertEnvIsLocked(); Set<String> indexFolders = new HashSet<>(); for (NodePath nodePath : nodePaths) { indexFolders.addAll(availableIndexFoldersForPath(nodePath, excludeIndexPathIdsPredicate)); } return indexFolders; }
/** * Deletes a shard data directory. Note: this method assumes that the shard * lock is acquired. This method will also attempt to acquire the write * locks for the shard's paths before deleting the data, but this is best * effort, as the lock is released before the deletion happens in order to * allow the folder to be deleted * * @param lock the shards lock * @throws IOException if an IOException occurs * @throws ElasticsearchException if the write.lock is not acquirable */ public void deleteShardDirectoryUnderLock(ShardLock lock, IndexSettings indexSettings) throws IOException { final ShardId shardId = lock.getShardId(); assert isShardLocked(shardId) : "shard " + shardId + " is not locked"; final Path[] paths = availableShardPaths(shardId); logger.trace("acquiring locks for {}, paths: [{}]", shardId, paths); acquireFSLockForPaths(indexSettings, paths); IOUtils.rm(paths); if (indexSettings.hasCustomDataPath()) { Path customLocation = resolveCustomLocation(indexSettings, shardId); logger.trace("acquiring lock for {}, custom path: [{}]", shardId, customLocation); acquireFSLockForPaths(indexSettings, customLocation); logger.trace("deleting custom shard {} directory [{}]", shardId, customLocation); IOUtils.rm(customLocation); } logger.trace("deleted shard {} directory, paths: [{}]", shardId, paths); assert assertPathsDoNotExist(paths); }
/** * Deletes an indexes data directory recursively iff all of the indexes * shards locks were successfully acquired. If any of the indexes shard directories can't be locked * non of the shards will be deleted * * @param index the index to delete * @param lockTimeoutMS how long to wait for acquiring the indices shard locks * @param indexSettings settings for the index being deleted * @throws IOException if any of the shards data directories can't be locked or deleted */ public void deleteIndexDirectorySafe(Index index, long lockTimeoutMS, IndexSettings indexSettings) throws IOException, ShardLockObtainFailedException { final List<ShardLock> locks = lockAllForIndex(index, indexSettings, lockTimeoutMS); try { deleteIndexDirectoryUnderLock(index, indexSettings); } finally { IOUtils.closeWhileHandlingException(locks); } }
Consumer<String> nodeIdConsumer = nodeNameExplicitlyDefined ? nodeId -> {} : nodeId -> registerDerivedNodeNameWithLogger(nodeIdToNodeName(nodeId)); nodeEnvironment = new NodeEnvironment(tmpSettings, environment, nodeIdConsumer); resourcesToClose.add(nodeEnvironment); } catch (IOException ex) { NODE_NAME_SETTING.get(tmpSettings), nodeEnvironment.nodeId()); } else { tmpSettings = Settings.builder() .put(tmpSettings) .put(NODE_NAME_SETTING.getKey(), nodeIdToNodeName(nodeEnvironment.nodeId())) .build(); logger.info("node name derived from node ID [{}]; set [{}] to override", nodeEnvironment.nodeId(), NODE_NAME_SETTING.getKey()); localNodeFactory = new LocalNodeFactory(settings, nodeEnvironment.nodeId());
nodeEnvironment = new NodeEnvironment(this.settings, this.environment); } catch (IOException ex) { throw new IllegalStateException("Failed to created node environment", ex); } finally { if (!success) { nodeEnvironment.close(); ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
private void markNodeDataDirsAsPendingForWipe(Node node) { assert Thread.holdsLock(this); NodeEnvironment nodeEnv = node.getNodeEnvironment(); if (nodeEnv.hasNodeFile()) { dataDirToClean.addAll(Arrays.asList(nodeEnv.nodeDataPaths())); } }
/** * Loads the global state, *without* index state, see {@link #loadFullState()} for that. */ MetaData loadGlobalState() throws IOException { return MetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); }
/** * This method tries to delete left-over shards where the index name has been reused but the UUID is different * to allow the new shard to be allocated. */ public static void deleteLeftoverShardDirectory(Logger logger, NodeEnvironment env, ShardLock lock, IndexSettings indexSettings) throws IOException { final String indexUUID = indexSettings.getUUID(); final Path[] paths = env.availableShardPaths(lock.getShardId()); for (Path path : paths) { // EMPTY is safe here because we never call namedObject ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, path); if (load != null) { if (load.indexUUID.equals(indexUUID) == false && IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID) == false) { logger.warn("{} deleting leftover shard on path: [{}] with a different index UUID", lock.getShardId(), path); assert Files.isDirectory(path) : path + " is not a directory"; NodeEnvironment.acquireFSLockForPaths(indexSettings, paths); IOUtils.rm(path); } } } }
private void deleteIndexStoreIfDeletionAllowed(final String reason, final Index index, final IndexSettings indexSettings, final IndexDeletionAllowedPredicate predicate) throws IOException { boolean success = false; try { // we are trying to delete the index store here - not a big deal if the lock can't be obtained // the store metadata gets wiped anyway even without the lock this is just best effort since // every shards deletes its content under the shard lock it owns. logger.debug("{} deleting index store reason [{}]", index, reason); if (predicate.apply(index, indexSettings)) { // its safe to delete all index metadata and shard data nodeEnv.deleteIndexDirectorySafe(index, 0, indexSettings); } success = true; } catch (LockObtainFailedException ex) { logger.debug(() -> new ParameterizedMessage("{} failed to delete index store - at least one shards is still locked", index), ex); } catch (Exception ex) { logger.warn(() -> new ParameterizedMessage("{} failed to delete index", index), ex); } finally { if (success == false) { addPendingDelete(index, indexSettings); } // this is a pure protection to make sure this index doesn't get re-imported as a dangling index. // we should in the future rather write a tombstone rather than wiping the metadata. MetaDataStateFormat.deleteMetaState(nodeEnv.indexPaths(index)); } }
/** * Tries to lock the given shards ID. A shard lock is required to perform any kind of * write operation on a shards data directory like deleting files, creating a new index writer * or recover from a different shard instance into it. If the shard lock can not be acquired * a {@link ShardLockObtainFailedException} is thrown. * * Note: this method will return immediately if the lock can't be acquired. * * @param id the shard ID to lock * @return the shard lock. Call {@link ShardLock#close()} to release the lock */ public ShardLock shardLock(ShardId id) throws ShardLockObtainFailedException { return shardLock(id, 0); }
/** * This method returns true if the current node is allowed to delete the given index. * This is the case if the index is deleted in the metadata or there is no allocation * on the local node and the index isn't on a shared file system. * @param index {@code Index} to check whether deletion is allowed * @param indexSettings {@code IndexSettings} for the given index * @return true if the index can be deleted on this node */ public boolean canDeleteIndexContents(Index index, IndexSettings indexSettings) { // index contents can be deleted if its an already closed index (so all its resources have // already been relinquished) final IndexService indexService = indexService(index); if (indexService == null && nodeEnv.hasNodeFile()) { return true; } return false; }
/** * Returns <code>ShardDeletionCheckResult</code> signaling whether the shards content for the given shard can be deleted. * * @param shardId the shard to delete. * @param indexSettings the shards's relevant {@link IndexSettings}. This is required to access the indexes settings etc. */ public ShardDeletionCheckResult canDeleteShardContent(ShardId shardId, IndexSettings indexSettings) { assert shardId.getIndex().equals(indexSettings.getIndex()); final IndexService indexService = indexService(shardId.getIndex()); if (nodeEnv.hasNodeFile()) { final boolean isAllocated = indexService != null && indexService.hasShard(shardId.id()); if (isAllocated) { return ShardDeletionCheckResult.STILL_ALLOCATED; // we are allocated - can't delete the shard } else if (indexSettings.hasCustomDataPath()) { // lets see if it's on a custom path (return false if the shared doesn't exist) // we don't need to delete anything that is not there return Files.exists(nodeEnv.resolveCustomLocation(indexSettings, shardId)) ? ShardDeletionCheckResult.FOLDER_FOUND_CAN_DELETE : ShardDeletionCheckResult.NO_FOLDER_FOUND; } else { // lets see if it's path is available (return false if the shared doesn't exist) // we don't need to delete anything that is not there return FileSystemUtils.exists(nodeEnv.availableShardPaths(shardId)) ? ShardDeletionCheckResult.FOLDER_FOUND_CAN_DELETE : ShardDeletionCheckResult.NO_FOLDER_FOUND; } } else { return ShardDeletionCheckResult.NO_LOCAL_STORAGE; } }
private boolean canDeleteShardContent(ShardId shardId, Settings indexSettings) { final IndexServiceInjectorPair indexServiceInjectorPair = this.indices.get(shardId.getIndex()); if (IndexMetaData.isOnSharedFilesystem(indexSettings) == false) { if (nodeEnv.hasNodeFile()) { boolean isAllocated = indexServiceInjectorPair != null && indexServiceInjectorPair .getIndexService().hasShard(shardId.getId()); if (isAllocated) { return false; // we are allocated - can't delete the shard } if (NodeEnvironment.hasCustomDataPath(indexSettings)) { // lets see if it's on a custom path (return false if the shared doesn't exist) // we don't need to delete anything that is not there return Files.exists(nodeEnv.resolveCustomLocation(indexSettings, shardId)); } else { // lets see if it's path is available (return false if the shared doesn't exist) // we don't need to delete anything that is not there return FileSystemUtils.exists(nodeEnv.availableShardPaths(shardId)); } } } else { logger.trace("{} skipping shard directory deletion due to shadow replicas", shardId); } return false; }
/** * Returns an array of all of the nodes data locations. * @throws IllegalStateException if the node is not configured to store local locations */ public Path[] nodeDataPaths() { assertEnvIsLocked(); Path[] paths = new Path[nodePaths.length]; for(int i=0;i<paths.length;i++) { paths[i] = nodePaths[i].path; } return paths; }
/** * Find all the shards for this index, returning a map of the {@code NodePath} to the number of shards on that path * @param index the index by which to filter shards * @return a map of NodePath to count of the shards for the index on that path * @throws IOException if an IOException occurs */ public Map<NodePath, Long> shardCountPerPath(final Index index) throws IOException { assert index != null; if (nodePaths == null || locks == null) { throw new IllegalStateException("node is not configured to store local location"); } assertEnvIsLocked(); final Map<NodePath, Long> shardCountPerPath = new HashMap<>(); final String indexUniquePathId = index.getUUID(); for (final NodePath nodePath : nodePaths) { Path indexLocation = nodePath.indicesPath.resolve(indexUniquePathId); if (Files.isDirectory(indexLocation)) { shardCountPerPath.put(nodePath, (long) findAllShardsForIndex(indexLocation, index).size()); } } return shardCountPerPath; }