/** Returns true if the file exists * (can be opened), false if it cannot be opened, and * (unlike Java's File.exists) throws IOException if * there's some unexpected error. */ static boolean slowFileExists(Directory dir, String fileName) throws IOException { try { dir.openInput(fileName, IOContext.DEFAULT).close(); return true; } catch (NoSuchFileException | FileNotFoundException e) { return false; } }
/** Retrieves the full index header from the provided {@link IndexInput}. * This throws {@link CorruptIndexException} if this file does * not appear to be an index file. */ public static byte[] readIndexHeader(IndexInput in) throws IOException { in.seek(0); final int actualHeader = in.readInt(); if (actualHeader != CODEC_MAGIC) { throw new CorruptIndexException("codec header mismatch: actual header=" + actualHeader + " vs expected header=" + CODEC_MAGIC, in); } String codec = in.readString(); in.readInt(); in.seek(in.getFilePointer() + StringHelper.ID_LENGTH); int suffixLength = in.readByte() & 0xFF; byte[] bytes = new byte[headerLength(codec) + StringHelper.ID_LENGTH + 1 + suffixLength]; in.seek(0); in.readBytes(bytes, 0, bytes.length); return bytes; }
/** Retrieves the full footer from the provided {@link IndexInput}. This throws * {@link CorruptIndexException} if this file does not have a valid footer. */ public static byte[] readFooter(IndexInput in) throws IOException { if (in.length() < footerLength()) { throw new CorruptIndexException("misplaced codec footer (file truncated?): length=" + in.length() + " but footerLength==" + footerLength(), in); } in.seek(in.length() - footerLength()); validateFooter(in); in.seek(in.length() - footerLength()); byte[] bytes = new byte[footerLength()]; in.readBytes(bytes, 0, bytes.length); return bytes; }
SkipBuffer(IndexInput input, int length) throws IOException { super("SkipBuffer on " + input); data = new byte[length]; pointer = input.getFilePointer(); input.readBytes(data, 0, length); }
priorE = exception; } finally { CodecUtil.checkFooter(input, priorE); vectorsStream = d.openInput(vectorsStreamFN, context); final String codecNameDat = formatName + CODEC_SFX_DAT; int version2 = CodecUtil.checkIndexHeader(vectorsStream, codecNameDat, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix); if (version != version2) { throw new CorruptIndexException("Version mismatch between stored fields index and data: " + version + " != " + version2, vectorsStream); assert CodecUtil.indexHeaderLength(codecNameDat, segmentSuffix) == vectorsStream.getFilePointer(); long pos = vectorsStream.getFilePointer(); vectorsStream.seek(maxPointer); numChunks = vectorsStream.readVLong(); numDirtyChunks = vectorsStream.readVLong(); if (numDirtyChunks > numChunks) { throw new CorruptIndexException("invalid chunk counts: dirty=" + numDirtyChunks + ", total=" + numChunks, vectorsStream); CodecUtil.retrieveChecksum(vectorsStream); vectorsStream.seek(pos); packedIntsVersion = vectorsStream.readVInt(); chunkSize = vectorsStream.readVInt(); decompressor = compressionMode.newDecompressor(); this.reader = new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, PACKED_BLOCK_SIZE, 0);
byte b = termsIn.readByte(); if (b != 0) { throw new CorruptIndexException("Index header pretends the index has auto-prefix terms: " + b, termsIn); final int numFields = termsIn.readVInt(); if (numFields < 0) { throw new CorruptIndexException("invalid numFields: " + numFields, termsIn); final int field = termsIn.readVInt(); final long numTerms = termsIn.readVLong(); if (numTerms <= 0) { throw new CorruptIndexException("Illegal numTerms for field number: " + field, termsIn); final long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS ? -1 : termsIn.readVLong(); final long sumDocFreq = termsIn.readVLong(); final int docCount = termsIn.readVInt(); final int longsSize = termsIn.readVInt(); if (longsSize < 0) { throw new CorruptIndexException("invalid longsSize for field: " + fieldInfo.name + ", longsSize=" + longsSize, termsIn); final long indexStartFP = indexIn.readVLong(); FieldReader previous = fields.put(fieldInfo.name, new FieldReader(this, fieldInfo, numTerms, rootCode, sumTotalTermFreq, sumDocFreq, docCount, indexIn.close(); success = true; } finally {
rawDocs.seek(index.getStartPointer(0)); int docID = 0; while (docID < maxDoc) { int base = rawDocs.readVInt(); if (base != docID) { throw new CorruptIndexException("invalid state: base=" + base + ", docID=" + docID, rawDocs); int bufferedDocs = rawDocs.readVInt(); indexWriter.writeIndex(bufferedDocs, vectorsStream.getFilePointer()); vectorsStream.writeVInt(docCount); // rebase vectorsStream.writeVInt(bufferedDocs); docID += bufferedDocs; docCount += bufferedDocs; throw new CorruptIndexException("invalid state: base=" + base + ", count=" + bufferedDocs + ", maxDoc=" + maxDoc, rawDocs); end = index.getStartPointer(docID); vectorsStream.copyBytes(rawDocs, end - rawDocs.getFilePointer()); if (rawDocs.getFilePointer() != matchingVectorsReader.getMaxPointer()) { throw new CorruptIndexException("invalid state: pos=" + rawDocs.getFilePointer() + ", max=" + matchingVectorsReader.getMaxPointer(), rawDocs);
IOException ioe = null; List<String> snapshotFiles = new ArrayList<>(); for(String file : dir.listAll()) { if (file.startsWith(SNAPSHOTS_PREFIX)) { long gen = Long.parseLong(file.substring(SNAPSHOTS_PREFIX.length())); snapshotFiles.add(file); Map<Long,Integer> m = new HashMap<>(); IndexInput in = dir.openInput(file, IOContext.DEFAULT); try { CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START); int count = in.readVInt(); for(int i=0;i<count;i++) { long commitGen = in.readVLong(); int refCount = in.readVInt(); m.put(commitGen, refCount); in.close();
private static void checksumFromLuceneFile(Directory directory, String file, Map<String, StoreFileMetaData> builder, Logger logger, Version version, boolean readFileAsHash) throws IOException { final String checksum; final BytesRefBuilder fileHash = new BytesRefBuilder(); try (IndexInput in = directory.openInput(file, IOContext.READONCE)) { final long length; try { length = in.length(); if (length < CodecUtil.footerLength()) { // truncated files trigger IAE if we seek negative... these files are really corrupted though throw new CorruptIndexException("Can't retrieve checksum from file: " + file + " file length must be >= " + CodecUtil.footerLength() + " but was: " + in.length(), in); } if (readFileAsHash) { // additional safety we checksum the entire file we read the hash for... final VerifyingIndexInput verifyingIndexInput = new VerifyingIndexInput(in); hashFile(fileHash, new InputStreamIndexInput(verifyingIndexInput, length), length); checksum = digestToString(verifyingIndexInput.verify()); } else { checksum = digestToString(CodecUtil.retrieveChecksum(in)); } } catch (Exception ex) { logger.debug(() -> new ParameterizedMessage("Can retrieve checksum from file [{}]", file), ex); throw ex; } builder.put(file, new StoreFileMetaData(file, length, checksum, version, fileHash.get())); } }
/** * Reads the state from a given file and compares the expected version against the actual version of * the state. */ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) throws IOException { try (Directory dir = newDirectory(file.getParent())) { try (IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) { // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. CodecUtil.checksumEntireFile(indexInput); CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION, STATE_FILE_VERSION); final XContentType xContentType = XContentType.values()[indexInput.readInt()]; if (xContentType != FORMAT) { throw new IllegalStateException("expected state in " + file + " to be " + FORMAT + " format but was " + xContentType); } long filePointer = indexInput.getFilePointer(); long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer; try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) { try (XContentParser parser = XContentFactory.xContent(FORMAT) .createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, new InputStreamIndexInput(slice, contentSize))) { return fromXContent(parser); } } } catch(CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) { // we trick this into a dedicated exception with the original stacktrace throw new CorruptStateException(ex); } } }
if ((start + length) * bytesPerDoc + CodecUtil.footerLength() > tempDir.fileLength(tempFileName)) { throw new IllegalArgumentException("requested slice is beyond the length of this file: start=" + start + " length=" + length + " bytesPerDoc=" + bytesPerDoc + " fileLength=" + tempDir.fileLength(tempFileName) + " tempFileName=" + tempFileName); if (start == 0 && length*bytesPerDoc == tempDir.fileLength(tempFileName) - CodecUtil.footerLength()) { in.seek(seekFP); countLeft = length; packedValue = new byte[packedBytesLength];
/** * Copies an existing {@code src} file from directory {@code from} * to a non-existent file {@code dest} in this directory. */ public void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException { boolean success = false; try (IndexInput is = from.openInput(src, context); IndexOutput os = createOutput(dest, context)) { os.copyBytes(is, is.length()); success = true; } finally { if (!success) { IOUtils.deleteFilesIgnoringExceptions(this, dest); } } }
boolean success = false; long expectedLength = CodecUtil.indexHeaderLength(Lucene50CompoundFormat.DATA_CODEC, ""); for(Map.Entry<String,FileEntry> ent : entries.entrySet()) { expectedLength += ent.getValue().length; expectedLength += CodecUtil.footerLength(); handle = directory.openInput(dataFileName, context); try { CodecUtil.checkIndexHeader(handle, Lucene50CompoundFormat.DATA_CODEC, version, version, si.getId(), ""); if (handle.length() != expectedLength) { throw new CorruptIndexException("length should be " + expectedLength + " bytes, but is " + handle.length() + " instead", handle);
private static void validateFooter(IndexInput in) throws IOException { long remaining = in.length() - in.getFilePointer(); long expected = footerLength(); if (remaining < expected) { throw new CorruptIndexException("misplaced codec footer (file truncated?): remaining=" + remaining + ", expected=" + expected + ", fp=" + in.getFilePointer(), in); } else if (remaining > expected) { throw new CorruptIndexException("misplaced codec footer (file extended?): remaining=" + remaining + ", expected=" + expected + ", fp=" + in.getFilePointer(), in); } final int magic = in.readInt(); if (magic != FOOTER_MAGIC) { throw new CorruptIndexException("codec footer mismatch (file truncated?): actual footer=" + magic + " vs expected footer=" + FOOTER_MAGIC, in); } final int algorithmID = in.readInt(); if (algorithmID != 0) { throw new CorruptIndexException("codec footer mismatch: unknown algorithmID: " + algorithmID, in); } }
/** * Computes a strong hash value for small files. Note that this method should only be used for files < 1MB */ public static BytesRef hashFile(Directory directory, String file) throws IOException { final BytesRefBuilder fileHash = new BytesRefBuilder(); try (final IndexInput in = directory.openInput(file, IOContext.READONCE)) { hashFile(fileHash, new InputStreamIndexInput(in, in.length()), in.length()); } return fileHash.get(); }
try (ChecksumIndexInput indexIn = readState.directory.openChecksumInput(indexFileName, readState.context)) { Throwable priorE = null; try { CodecUtil.checkIndexHeader(indexIn, Lucene60PointsFormat.META_CODEC_NAME, Lucene60PointsFormat.INDEX_VERSION_START, priorE = t; } finally { CodecUtil.checkFooter(indexIn, priorE); Lucene60PointsFormat.DATA_EXTENSION); boolean success = false; dataIn = readState.directory.openInput(dataFileName, readState.context); try { CodecUtil.checkIndexHeader(dataIn, Lucene60PointsFormat.DATA_CODEC_NAME, Lucene60PointsFormat.DATA_VERSION_START, int fieldNumber = ent.getKey(); long fp = ent.getValue(); dataIn.seek(fp); BKDReader reader = new BKDReader(dataIn); readers.put(fieldNumber, reader);
/** * Returns (but does not validate) the checksum previously written by {@link #checkFooter}. * @return actual checksum value * @throws IOException if the footer is invalid */ public static long retrieveChecksum(IndexInput in) throws IOException { if (in.length() < footerLength()) { throw new CorruptIndexException("misplaced codec footer (file truncated?): length=" + in.length() + " but footerLength==" + footerLength(), in); } in.seek(in.length() - footerLength()); validateFooter(in); return readCRC(in); }