/** * Serializes file info into JSON * * @param file file info * @param builder XContent builder * @param params parameters */ public static void toXContent(FileInfo file, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(); builder.field(NAME, file.name); builder.field(PHYSICAL_NAME, file.metadata.name()); builder.field(LENGTH, file.metadata.length()); if (file.metadata.checksum().equals(UNKNOWN_CHECKSUM) == false) { builder.field(CHECKSUM, file.metadata.checksum()); } if (file.partSize != null) { builder.field(PART_SIZE, file.partSize.getBytes()); } if (file.metadata.writtenBy() != null) { builder.field(WRITTEN_BY, file.metadata.writtenBy()); } if (file.metadata.hash() != null && file.metadata().hash().length > 0) { BytesRef br = file.metadata.hash(); builder.field(META_HASH, br.bytes, br.offset, br.length); } builder.endObject(); }
/** * Read from a stream. */ public MetadataSnapshot(StreamInput in) throws IOException { final int size = in.readVInt(); Map<String, StoreFileMetaData> metadata = new HashMap<>(); for (int i = 0; i < size; i++) { StoreFileMetaData meta = new StoreFileMetaData(in); metadata.put(meta.name(), meta); } Map<String, String> commitUserData = new HashMap<>(); int num = in.readVInt(); for (int i = num; i > 0; i--) { commitUserData.put(in.readString(), in.readString()); } this.metadata = unmodifiableMap(metadata); this.commitUserData = unmodifiableMap(commitUserData); this.numDocs = in.readLong(); assert metadata.isEmpty() || numSegmentFiles() == 1 : "numSegmentFiles: " + numSegmentFiles(); }
private static long computeMatchingBytes(TransportNodesListShardStoreMetaData.StoreFilesMetaData primaryStore, TransportNodesListShardStoreMetaData.StoreFilesMetaData storeFilesMetaData) { String primarySyncId = primaryStore.syncId(); String replicaSyncId = storeFilesMetaData.syncId(); // see if we have a sync id we can make use of if (replicaSyncId != null && replicaSyncId.equals(primarySyncId)) { return Long.MAX_VALUE; } else { long sizeMatched = 0; for (StoreFileMetaData storeFileMetaData : storeFilesMetaData) { String metaDataFileName = storeFileMetaData.name(); if (primaryStore.fileExists(metaDataFileName) && primaryStore.file(metaDataFileName).isSame(storeFileMetaData)) { sizeMatched += storeFileMetaData.length(); } } return sizeMatched; } }
@Override public void verify() throws IOException { String footerDigest = null; if (metadata.checksum().equals(actualChecksum) && writtenBytes == metadata.length()) { ByteArrayIndexInput indexInput = new ByteArrayIndexInput("checksum", this.footerChecksum); footerDigest = digestToString(indexInput.readLong()); if (metadata.checksum().equals(footerDigest)) { return; } } throw new CorruptIndexException("verification failed (hardware problem?) : expected=" + metadata.checksum() + " actual=" + actualChecksum + " footer=" + footerDigest +" writtenLength=" + writtenBytes + " expectedLength=" + metadata.length() + " (resource=" + metadata.toString() + ")", "VerifyingIndexOutput(" + metadata.name() + ")"); }
private void readAndCompareChecksum() throws IOException { actualChecksum = digestToString(getChecksum()); if (!metadata.checksum().equals(actualChecksum)) { throw new CorruptIndexException("checksum failed (hardware problem?) : expected=" + metadata.checksum() + " actual=" + actualChecksum + " (resource=" + metadata.toString() + ")", "VerifyingIndexOutput(" + metadata.name() + ")"); } }
public static void checkIntegrity(final StoreFileMetaData md, final Directory directory) throws IOException { try (IndexInput input = directory.openInput(md.name(), IOContext.READONCE)) { if (input.length() != md.length()) { // first check the length no matter how old this file is throw new CorruptIndexException("expected length=" + md.length() + " != actual length: " + input.length() + " : file truncated?", input); } // throw exception if the file is corrupt String checksum = Store.digestToString(CodecUtil.checksumEntireFile(input)); // throw exception if metadata is inconsistent if (!checksum.equals(md.checksum())) { throw new CorruptIndexException("inconsistent metadata: lucene checksum=" + checksum + ", metadata checksum=" + md.checksum(), input); } } }
if (recoveryDiff.missing.isEmpty()) { for (StoreFileMetaData meta : recoveryDiff.different) { StoreFileMetaData local = targetMetaData.get(meta.name()); StoreFileMetaData remote = sourceMetaData.get(meta.name()); final boolean same = local.isSame(remote); local.checksum() == null && remote.checksum() == null && local.hash().equals(remote.hash()) && local.length() == remote.length()); final boolean consistent = hashAndLengthEqual || same; if (consistent == false) {
final void verifyAfterCleanup(MetadataSnapshot sourceMetaData, MetadataSnapshot targetMetaData) { final RecoveryDiff recoveryDiff = targetMetaData.recoveryDiff(sourceMetaData); if (recoveryDiff.identical.size() != recoveryDiff.size()) { if (recoveryDiff.missing.isEmpty()) { for (StoreFileMetaData meta : recoveryDiff.different) { StoreFileMetaData local = targetMetaData.get(meta.name()); StoreFileMetaData remote = sourceMetaData.get(meta.name()); // if we have different files then they must have no checksums; otherwise something went wrong during recovery. // we have that problem when we have an empty index is only a segments_1 file so we can't tell if it's a Lucene 4.8 file // and therefore no checksum is included. That isn't a problem since we simply copy it over anyway but those files // come out as different in the diff. That's why we have to double check here again if the rest of it matches. // all is fine this file is just part of a commit or a segment that is different if (local.isSame(remote) == false) { logger.debug("Files are different on the recovery target: {} ", recoveryDiff); throw new IllegalStateException("local version: " + local + " is different from remote version after recovery: " + remote, null); } } } else { logger.debug("Files are missing on the recovery target: {} ", recoveryDiff); throw new IllegalStateException("Files are missing on the recovery target: [different=" + recoveryDiff.different + ", missing=" + recoveryDiff.missing + ']', null); } } }
/** * File length * * @return file length */ public long length() { return metadata.length(); }
throw new ElasticsearchParseException("missing checksum for name [" + name + "]"); return new FileInfo(name, new StoreFileMetaData(physicalName, length, checksum, writtenBy, metaHash), partSize);
/** * Returns file md5 checksum provided by {@link org.elasticsearch.index.store.Store} * * @return file checksum */ public String checksum() { return metadata.checksum(); }
/** * Checks if a file in a store is the same file * * @param md file in a store * @return true if file in a store this this file have the same checksum and length */ public boolean isSame(StoreFileMetaData md) { return metadata.isSame(md); }
public IndexInput openVerifyingInput(String filename, IOContext context, StoreFileMetaData metadata) throws IOException { if (metadata.hasLegacyChecksum() || metadata.checksum() == null) { logger.debug("open legacy input for {}", filename); return directory().openInput(filename, context); } assert metadata.writtenBy() != null; assert metadata.writtenBy().onOrAfter(Version.LUCENE_4_8_0); return new VerifyingIndexInput(directory().openInput(filename, context)); }
public IndexInput openVerifyingInput(String filename, IOContext context, StoreFileMetaData metadata) throws IOException { assert metadata.writtenBy() != null; return new VerifyingIndexInput(directory().openInput(filename, context)); }
/** * Returns original file name * * @return original file name */ public String physicalName() { return metadata.name(); }
@Override public void writeTo(StreamOutput out) throws IOException { out.writeVInt(this.metadata.size()); for (StoreFileMetaData meta : this) { meta.writeTo(out); } out.writeVInt(commitUserData.size()); for (Map.Entry<String, String> entry : commitUserData.entrySet()) { out.writeString(entry.getKey()); out.writeString(entry.getValue()); } out.writeLong(numDocs); }
/** * This is a BWC layer to ensure we update the snapshots metadata with the corresponding hashes before we compare them. * The new logic for StoreFileMetaData reads the entire {@code .si} and {@code segments.n} files to strengthen the * comparison of the files on a per-segment / per-commit level. */ private static void maybeRecalculateMetadataHash(final BlobContainer blobContainer, final BlobStoreIndexShardSnapshot.FileInfo fileInfo, Store.MetadataSnapshot snapshot) throws Exception { final StoreFileMetaData metadata; if (fileInfo != null && (metadata = snapshot.get(fileInfo.physicalName())) != null) { if (metadata.hash().length > 0 && fileInfo.metadata().hash().length == 0) { // we have a hash - check if our repo has a hash too otherwise we have // to calculate it. // we might have multiple parts even though the file is small... make sure we read all of it. try (InputStream stream = new PartSliceStream(blobContainer, fileInfo)) { BytesRefBuilder builder = new BytesRefBuilder(); Store.MetadataSnapshot.hashFile(builder, stream, fileInfo.length()); BytesRef hash = fileInfo.metadata().hash(); // reset the file infos metadata hash assert hash.length == 0; hash.bytes = builder.bytes(); hash.offset = 0; hash.length = builder.length(); } } } }
public static StoreFileMetaData readStoreFileMetaData(StreamInput in) throws IOException { StoreFileMetaData md = new StoreFileMetaData(); md.readFrom(in); return md; }