@Override public void checkIntegrity() throws IOException { if (docIn != null) { CodecUtil.checksumEntireFile(docIn); } if (posIn != null) { CodecUtil.checksumEntireFile(posIn); } if (payIn != null) { CodecUtil.checksumEntireFile(payIn); } }
@Override public void checkIntegrity() throws IOException { CodecUtil.checksumEntireFile(vectorsStream); }
@Override public void checkIntegrity() throws IOException { CodecUtil.checksumEntireFile(data); }
@Override public void checkIntegrity() throws IOException { CodecUtil.checksumEntireFile(fieldsStream); }
@Override public void checkIntegrity() throws IOException { CodecUtil.checksumEntireFile(data); }
@Override public void checkIntegrity() throws IOException { CodecUtil.checksumEntireFile(dataIn); }
@Override public void checkIntegrity() throws IOException { // term dictionary CodecUtil.checksumEntireFile(termsIn); // postings postingsReader.checkIntegrity(); }
@Override protected Object doBody(String segmentFileName) throws IOException { try (IndexInput input = directory.openInput(segmentFileName, IOContext.READ)) { CodecUtil.checksumEntireFile(input); } return null; } }.run();
indexIn = state.directory.openInput(indexName, state.context); CodecUtil.checkIndexHeader(indexIn, TERMS_INDEX_CODEC_NAME, version, version, state.segmentInfo.getId(), state.segmentSuffix); CodecUtil.checksumEntireFile(indexIn);
public static Checkpoint read(Path path) throws IOException { try (Directory dir = new SimpleFSDirectory(path.getParent())) { try (IndexInput indexInput = dir.openInput(path.getFileName().toString(), IOContext.DEFAULT)) { // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. CodecUtil.checksumEntireFile(indexInput); final int fileVersion = CodecUtil.checkHeader(indexInput, CHECKPOINT_CODEC, INITIAL_VERSION, CURRENT_VERSION); if (fileVersion == INITIAL_VERSION) { assert indexInput.length() == V1_FILE_SIZE : indexInput.length(); return Checkpoint.readCheckpointV5_0_0(indexInput); } else if (fileVersion == VERSION_6_0_0) { assert indexInput.length() == V2_FILE_SIZE : indexInput.length(); return Checkpoint.readCheckpointV6_0_0(indexInput); } else { assert fileVersion == CURRENT_VERSION : fileVersion; assert indexInput.length() == V3_FILE_SIZE : indexInput.length(); return Checkpoint.readCheckpointV6_4_0(indexInput); } } } }
/** * Reads the state from a given file and compares the expected version against the actual version of * the state. */ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) throws IOException { try (Directory dir = newDirectory(file.getParent())) { try (IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) { // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. CodecUtil.checksumEntireFile(indexInput); CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION, STATE_FILE_VERSION); final XContentType xContentType = XContentType.values()[indexInput.readInt()]; if (xContentType != FORMAT) { throw new IllegalStateException("expected state in " + file + " to be " + FORMAT + " format but was " + xContentType); } long filePointer = indexInput.getFilePointer(); long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer; try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) { try (XContentParser parser = XContentFactory.xContent(FORMAT) .createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, new InputStreamIndexInput(slice, contentSize))) { return fromXContent(parser); } } } catch(CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) { // we trick this into a dedicated exception with the original stacktrace throw new CorruptStateException(ex); } } }
/** * Reads blob with specified name without resolving the blobName using using {@link #blobName} method. * * @param blobContainer blob container * @param blobName blob name */ public T readBlob(BlobContainer blobContainer, String blobName) throws IOException { try (InputStream inputStream = blobContainer.readBlob(blobName)) { ByteArrayOutputStream out = new ByteArrayOutputStream(); Streams.copy(inputStream, out); final byte[] bytes = out.toByteArray(); final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")"; try (ByteArrayIndexInput indexInput = new ByteArrayIndexInput(resourceDesc, bytes)) { CodecUtil.checksumEntireFile(indexInput); CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION); long filePointer = indexInput.getFilePointer(); long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer; BytesReference bytesReference = new BytesArray(bytes, (int) filePointer, (int) contentSize); return read(bytesReference); } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) { // we trick this into a dedicated exception with the original stacktrace throw new CorruptStateException(ex); } } }
public static void checkIntegrity(final StoreFileMetaData md, final Directory directory) throws IOException { try (IndexInput input = directory.openInput(md.name(), IOContext.READONCE)) { if (input.length() != md.length()) { // first check the length no matter how old this file is throw new CorruptIndexException("expected length=" + md.length() + " != actual length: " + input.length() + " : file truncated?", input); } // throw exception if the file is corrupt String checksum = Store.digestToString(CodecUtil.checksumEntireFile(input)); // throw exception if metadata is inconsistent if (!checksum.equals(md.checksum())) { throw new CorruptIndexException("inconsistent metadata: lucene checksum=" + checksum + ", metadata checksum=" + md.checksum(), input); } } }
@Override public void checkIntegrity() throws IOException { CodecUtil.checksumEntireFile(fieldsStream); }
@Override public void checkIntegrity() throws IOException { // term dictionary CodecUtil.checksumEntireFile(termsIn); // postings postingsReader.checkIntegrity(); }
@Override public void checkIntegrity() throws IOException { // term dictionary CodecUtil.checksumEntireFile(termsIn); // postings postingsReader.checkIntegrity(); }
@Override protected Object doBody(String segmentFileName) throws IOException { try (IndexInput input = directory.openInput(segmentFileName, IOContext.READ)) { CodecUtil.checksumEntireFile(input); } return null; } }.run();
@Override public void checkIntegrity() throws IOException { CodecUtil.checksumEntireFile(data.clone()); }
@Override public void checkIntegrity() throws IOException { // verify terms CodecUtil.checksumEntireFile(in); // verify postings postingsReader.checkIntegrity(); } }
@Override public void checkIntegrity() throws IOException { // term dictionary CodecUtil.checksumEntireFile(in); // postings postingsReader.checkIntegrity(); }