/** * Creates a CharArraySet from a path. * * @param stopwords * the stopwords file to load * @return a CharArraySet containing the distinct stopwords from the given * file * @throws IOException * if loading the stopwords throws an {@link IOException} */ protected static CharArraySet loadStopwordSet(Path stopwords) throws IOException { Reader reader = null; try { reader = Files.newBufferedReader(stopwords, StandardCharsets.UTF_8); return WordlistLoader.getWordSet(reader); } finally { IOUtils.close(reader); } }
Lucene70NormsProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException { maxDoc = state.segmentInfo.maxDoc(); String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension); int version = -1; try (ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context)) { Throwable priorE = null; try { version = CodecUtil.checkIndexHeader(in, metaCodec, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); readFields(in, state.fieldInfos); } catch (Throwable exception) { priorE = exception; } finally { CodecUtil.checkFooter(in, priorE); String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension); data = state.directory.openInput(dataName, state.context); boolean success = false; try { final int version2 = CodecUtil.checkIndexHeader(data, dataCodec, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); if (version != version2) { throw new CorruptIndexException("Format versions mismatch: meta=" + version + ",data=" + version2, data); } finally { if (!success) { IOUtils.closeWhileHandlingException(this.data);
boolean success = false; try { input = getBufferedReader(IOUtils.getDecodingReader(stream, charset)); } finally { if (success) { IOUtils.close(input); } else { IOUtils.closeWhileHandlingException(input);
/** * Opens a Reader for the given resource using a {@link CharsetDecoder}. * Unlike Java's defaults this reader will throw an exception if your it detects * the read charset doesn't match the expected {@link Charset}. * <p> * Decoding readers are useful to load configuration files, stopword lists or synonym files * to detect character set problems. However, it's not recommended to use as a common purpose * reader. * @param clazz the class used to locate the resource * @param resource the resource name to load * @param charSet the expected charset * @return a reader to read the given file * */ public static Reader getDecodingReader(Class<?> clazz, String resource, Charset charSet) throws IOException { InputStream stream = null; boolean success = false; try { stream = clazz .getResourceAsStream(resource); final Reader reader = getDecodingReader(stream, charSet); success = true; return reader; } finally { if (!success) { IOUtils.close(stream); } } }
@Override public void close() throws IOException { // TODO: add a finish() at least to PushBase? DV too...? boolean success = false; try { if (docOut != null) { CodecUtil.writeFooter(docOut); } if (posOut != null) { CodecUtil.writeFooter(posOut); } if (payOut != null) { CodecUtil.writeFooter(payOut); } success = true; } finally { if (success) { IOUtils.close(docOut, posOut, payOut); } else { IOUtils.closeWhileHandlingException(docOut, posOut, payOut); } docOut = posOut = payOut = null; } } }
synchronized private void persist() throws IOException { String fileName = SNAPSHOTS_PREFIX + nextWriteGen; IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT); boolean success = false; try { CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT); out.writeVInt(refCounts.size()); for(Entry<Long,Integer> ent : refCounts.entrySet()) { out.writeVLong(ent.getKey()); out.writeVInt(ent.getValue()); } success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(out); IOUtils.deleteFilesIgnoringExceptions(dir, fileName); } else { IOUtils.close(out); } } dir.sync(Collections.singletonList(fileName)); if (nextWriteGen > 0) { String lastSaveFile = SNAPSHOTS_PREFIX + (nextWriteGen-1); // exception OK: likely it didn't exist IOUtils.deleteFilesIgnoringExceptions(dir, lastSaveFile); } nextWriteGen++; }
Path rules = Files.createTempFile(tmpFile, "es", "sb"); Files.write(rules, Collections.singleton(SANDBOX_RULES)); int ret = libc_mac.sandbox_init(rules.toAbsolutePath().toString(), SANDBOX_NAMED, errorRef); throw e; logger.debug("OS X seatbelt initialization successful"); success = true; } finally { if (success) { Files.delete(rules); } else { IOUtils.deleteFilesIgnoringExceptions(rules);
Files.createDirectories(lockDir); Path lockFile = lockDir.resolve(lockName); Files.createFile(lockFile); } catch (IOException ignore) { realPath = lockFile.toRealPath(); } catch (IOException e) { final FileTime creationTime = Files.readAttributes(realPath, BasicFileAttributes.class).creationTime(); if (LOCK_HELD.add(realPath.toString())) { FileChannel channel = null; FileLock lock = null; IOUtils.closeWhileHandlingException(channel); // TODO: addSuppressed
@Override public void handle(ChannelReference channelReference) { if (isReferencedGeneration(channelReference.getGeneration()) == false) { Path translogPath = channelReference.getPath(); assert channelReference.getPath().getParent().equals(location) : "translog files must be in the location folder: " + location + " but was: " + translogPath; // if the given translogPath is not the current we can safely delete the file since all references are released logger.trace("delete translog file - not referenced and not current anymore {}", translogPath); IOUtils.deleteFilesIgnoringExceptions(translogPath); IOUtils.deleteFilesIgnoringExceptions(translogPath.resolveSibling(getCommitCheckpointFileName(channelReference.getGeneration()))); } try (DirectoryStream<Path> stream = Files.newDirectoryStream(location)) { for (Path path : stream) { Matcher matcher = PARSE_STRICT_ID_PATTERN.matcher(path.getFileName().toString()); if (matcher.matches()) { long generation = Long.parseLong(matcher.group(1)); if (isReferencedGeneration(generation) == false) { logger.trace("delete translog file - not referenced and not current anymore {}", path); IOUtils.deleteFilesIgnoringExceptions(path); IOUtils.deleteFilesIgnoringExceptions(path.resolveSibling(getCommitCheckpointFileName(generation))); } } } } catch (IOException e) { logger.warn("failed to delete unreferenced translog files", e); } } }
@Override public void move(String source, String target) throws IOException { Path sourcePath = path.resolve(source); Path targetPath = path.resolve(target); // If the target file exists then Files.move() behaviour is implementation specific // the existing file might be replaced or this method fails by throwing an IOException. assert !Files.exists(targetPath); Files.move(sourcePath, targetPath, StandardCopyOption.ATOMIC_MOVE); IOUtils.fsync(path, true); } }
private Directory createDirectory(boolean eraseIndex, String dirName, String dirParam) throws IOException { if ("FSDirectory".equals(config.get(dirParam,"RAMDirectory"))) { Path workDir = Paths.get(config.get("work.dir","work")); Path indexDir = workDir.resolve(dirName); if (eraseIndex && Files.exists(indexDir)) { IOUtils.rm(indexDir); } Files.createDirectories(indexDir); return FSDirectory.open(indexDir); } return new RAMDirectory(); }
@Override public void writeBlob(String blobName, BytesReference data) throws IOException { final Path file = path.resolve(blobName); try (OutputStream outputStream = Files.newOutputStream(file)) { data.writeTo(outputStream); } IOUtils.fsync(file, false); IOUtils.fsync(path, true); }
oldCurrent.sync(); currentCommittingTranslog = current.immutableReader(); Path checkpoint = location.resolve(CHECKPOINT_FILE_NAME); assert Checkpoint.read(checkpoint).generation == currentCommittingTranslog.getGeneration(); Path commitCheckpoint = location.resolve(getCommitCheckpointFileName(currentCommittingTranslog.getGeneration())); Files.copy(checkpoint, commitCheckpoint); IOUtils.fsync(commitCheckpoint, false); IOUtils.fsync(commitCheckpoint.getParent(), true); view.onNewTranslog(currentCommittingTranslog.clone(), current.newReaderFromWriter()); IOUtils.close(oldCurrent); logger.trace("current translog set to [{}]", current.getGeneration()); assert oldCurrent.syncNeeded() == false : "old translog oldCurrent must not need a sync"; IOUtils.closeWhileHandlingException(this); // tragic event throw t;
@Override public void renameFile(String source, String dest) throws IOException { ensureOpen(); Files.move(directory.resolve(source), directory.resolve(dest), StandardCopyOption.ATOMIC_MOVE); // TODO: should we move directory fsync to a separate 'syncMetadata' method? // for example, to improve listCommits(), IndexFileDeleter could also call that after deleting segments_Ns IOUtils.fsync(directory, true); }
flagLookup.add(new BytesRef()); // no flags -> ord 0 Path aff = Files.createTempFile(tempPath, "affix", "aff"); OutputStream out = new BufferedOutputStream(Files.newOutputStream(aff)); InputStream aff1 = null; InputStream aff2 = null; aff1 = new BufferedInputStream(Files.newInputStream(aff)); String encoding = getDictionaryEncoding(aff1); IOUtils.closeWhileHandlingException(out, aff1, aff2); if (success) { Files.delete(aff); } else { IOUtils.deleteFilesIgnoringExceptions(aff);
/** * Deletes all meta state directories recursively for the given data locations * @param dataLocations the data location to delete */ public static void deleteMetaState(Path... dataLocations) throws IOException { Path[] stateDirectories = new Path[dataLocations.length]; for (int i = 0; i < dataLocations.length; i++) { stateDirectories[i] = dataLocations[i].resolve(STATE_DIR_NAME); } IOUtils.rm(stateDirectories); } }
/** * Moves the index folder found in <code>source</code> to <code>target</code> */ void upgrade(final Index index, final Path source, final Path target) throws IOException { boolean success = false; try { Files.move(source, target, StandardCopyOption.ATOMIC_MOVE); success = true; } catch (NoSuchFileException | FileNotFoundException exception) { // thrown when the source is non-existent because the folder was renamed // by another node (shared FS) after we checked if the target exists logger.error((Supplier<?>) () -> new ParameterizedMessage("multiple nodes trying to upgrade [{}] in parallel, retry " + "upgrading with single node", target), exception); throw exception; } finally { if (success) { logger.info("{} moved from [{}] to [{}]", index, source, target); logger.trace("{} syncing directory [{}]", index, target); IOUtils.fsync(target, true); } } }
/** * Deletes one or more files or directories (and everything underneath it). * * @throws IOException if any of the given files (or their subhierarchy files in case * of directories) cannot be removed. */ public static void rm(Path... locations) throws IOException { LinkedHashMap<Path,Throwable> unremoved = rm(new LinkedHashMap<Path,Throwable>(), locations); if (!unremoved.isEmpty()) { StringBuilder b = new StringBuilder("Could not remove the following files (in the order of attempts):\n"); for (Map.Entry<Path,Throwable> kv : unremoved.entrySet()) { b.append(" ") .append(kv.getKey().toAbsolutePath()) .append(": ") .append(kv.getValue()) .append("\n"); } throw new IOException(b.toString()); } }
/** * This method tries to delete left-over shards where the index name has been reused but the UUID is different * to allow the new shard to be allocated. */ public static void deleteLeftoverShardDirectory(Logger logger, NodeEnvironment env, ShardLock lock, IndexSettings indexSettings) throws IOException { final String indexUUID = indexSettings.getUUID(); final Path[] paths = env.availableShardPaths(lock.getShardId()); for (Path path : paths) { // EMPTY is safe here because we never call namedObject ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, path); if (load != null) { if (load.indexUUID.equals(indexUUID) == false && IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID) == false) { logger.warn("{} deleting leftover shard on path: [{}] with a different index UUID", lock.getShardId(), path); assert Files.isDirectory(path) : path + " is not a directory"; NodeEnvironment.acquireFSLockForPaths(indexSettings, paths); IOUtils.rm(path); } } } }