String hfileName = storeFile.getName(); if (storeFile.hasReference()) { Reference reference = Reference.convert(storeFile.getReference()); reference.write(fs, outPath); } else {
final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { TableName table = TableName.valueOf(snapshot.getTable()); String fileName = storeFile.getName(); if (storeFile.hasReference()) { if (storeFile.hasFileSize() && storeFile.getFileSize() != fstat.getLen()) { String msg = "hfile: " + fileName + " size does not match with the expected one. " + " found=" + fstat.getLen() + " expected=" + storeFile.getFileSize(); LOG.error(msg); throw new CorruptedSnapshotException(msg,
SnapshotRegionManifest.StoreFile.newBuilder(); sfManifest.setName(storeFile.getPath().getName()); family.addStoreFiles(sfManifest.build());
for (int j = 0; j < 100; ++j) { SnapshotRegionManifest.StoreFile.Builder sfManifest = SnapshotRegionManifest.StoreFile.newBuilder(); sfManifest.setName(String.format("%032d", i)); sfManifest.setFileSize((1 + i) * (1 + i) * 1024);
private Path createRegionManifest() throws IOException { byte[] startKey = Bytes.toBytes("AAAAAA"); byte[] stopKey = Bytes.toBytes("BBBBBB"); HRegionInfo regionInfo = new HRegionInfo(TABLE_NAME, startKey, stopKey, false); SnapshotRegionManifest.Builder dataRegionManifestBuilder = SnapshotRegionManifest.newBuilder(); dataRegionManifestBuilder.setRegionInfo(HRegionInfo.convert(regionInfo)); for (ColumnFamilyDescriptor hcd: builder.getTableDescriptor().getColumnFamilies()) { SnapshotRegionManifest.FamilyFiles.Builder family = SnapshotRegionManifest.FamilyFiles.newBuilder(); family.setFamilyName(UnsafeByteOperations.unsafeWrap(hcd.getName())); for (int j = 0; j < TEST_NUM_REGIONFILES; ++j) { SnapshotRegionManifest.StoreFile.Builder sfManifest = SnapshotRegionManifest.StoreFile.newBuilder(); sfManifest.setName(String.format("%064d", j)); sfManifest.setFileSize(j * 1024); family.addStoreFiles(sfManifest.build()); } dataRegionManifestBuilder.addFamilyFiles(family.build()); } SnapshotRegionManifest manifest = dataRegionManifestBuilder.build(); Path regionPath = new Path(snapshotDir, SnapshotManifestV2.SNAPSHOT_MANIFEST_PREFIX + regionInfo.getEncodedName()); FSDataOutputStream stream = fs.create(regionPath); try { manifest.writeTo(stream); } finally { stream.close(); } return regionPath; }
@java.lang.Override public boolean equals(final java.lang.Object obj) { if (obj == this) { return true; } if (!(obj instanceof org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile)) { return super.equals(obj); } org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile other = (org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile) obj; boolean result = true; result = result && (hasName() == other.hasName()); if (hasName()) { result = result && getName() .equals(other.getName()); } result = result && (hasReference() == other.hasReference()); if (hasReference()) { result = result && getReference() .equals(other.getReference()); } result = result && (hasFileSize() == other.hasFileSize()); if (hasFileSize()) { result = result && (getFileSize() == other.getFileSize()); } result = result && unknownFields.equals(other.unknownFields); return result; }
@java.lang.Override public int hashCode() { if (memoizedHashCode != 0) { return memoizedHashCode; } int hash = 41; hash = (19 * hash) + getDescriptor().hashCode(); if (hasName()) { hash = (37 * hash) + NAME_FIELD_NUMBER; hash = (53 * hash) + getName().hashCode(); } if (hasReference()) { hash = (37 * hash) + REFERENCE_FIELD_NUMBER; hash = (53 * hash) + getReference().hashCode(); } if (hasFileSize()) { hash = (37 * hash) + FILE_SIZE_FIELD_NUMBER; hash = (53 * hash) + org.apache.hbase.thirdparty.com.google.protobuf.Internal.hashLong( getFileSize()); } hash = (29 * hash) + unknownFields.hashCode(); memoizedHashCode = hash; return hash; }
@Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { // for storeFile.hasReference() case, copied as part of the manifest if (!storeFile.hasReference()) { String region = regionInfo.getEncodedName(); String hfile = storeFile.getName(); Path path = HFileLink.createPath(table, region, family, hfile); SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder() .setType(SnapshotFileInfo.Type.HFILE) .setHfile(path.toString()) .build(); long size; if (storeFile.hasFileSize()) { size = storeFile.getFileSize(); } else { size = HFileLink.buildFromHFileLinkPattern(conf, path).getFileStatus(fs).getLen(); } files.add(new Pair<>(fileInfo, size)); } } });
public int getSerializedSize() { int size = memoizedSize; if (size != -1) return size; size = 0; if (((bitField0_ & 0x00000001) == 0x00000001)) { size += org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessageV3.computeStringSize(1, name_); } if (((bitField0_ & 0x00000002) == 0x00000002)) { size += org.apache.hbase.thirdparty.com.google.protobuf.CodedOutputStream .computeMessageSize(2, getReference()); } if (((bitField0_ & 0x00000004) == 0x00000004)) { size += org.apache.hbase.thirdparty.com.google.protobuf.CodedOutputStream .computeUInt64Size(3, fileSize_); } size += unknownFields.getSerializedSize(); memoizedSize = size; return size; }
@Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { // for storeFile.hasReference() case, copied as part of the manifest if (!storeFile.hasReference()) { String region = regionInfo.getEncodedName(); String hfile = storeFile.getName(); Path path = HFileLink.createPath(table, region, family, hfile); SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder() .setType(SnapshotFileInfo.Type.HFILE) .setHfile(path.toString()) .build(); long size; if (storeFile.hasFileSize()) { size = storeFile.getFileSize(); } else { size = HFileLink.buildFromHFileLinkPattern(conf, path).getFileStatus(fs).getLen(); } files.add(new Pair<>(fileInfo, size)); } } });
public final boolean isInitialized() { byte isInitialized = memoizedIsInitialized; if (isInitialized == 1) return true; if (isInitialized == 0) return false; if (!hasName()) { memoizedIsInitialized = 0; return false; } if (hasReference()) { if (!getReference().isInitialized()) { memoizedIsInitialized = 0; return false; } } memoizedIsInitialized = 1; return true; }
public Builder mergeFrom(org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile other) { if (other == org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile.getDefaultInstance()) return this; if (other.hasName()) { bitField0_ |= 0x00000001; name_ = other.name_; onChanged(); } if (other.hasReference()) { mergeReference(other.getReference()); } if (other.hasFileSize()) { setFileSize(other.getFileSize()); } this.mergeUnknownFields(other.unknownFields); onChanged(); return this; }
@Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) return; String hfile = storeFile.getName(); snapshotFiles.add(hfile); if (storeFile.hasReference()) { // Nothing to do here, we have already the reference embedded } else { verifyNonEmptyFile(new Path(exportedArchive, new Path(FSUtils.getTableDir(new Path("./"), tableName), new Path(regionInfo.getEncodedName(), new Path(family, hfile))))); } }
public Builder newBuilderForType() { return newBuilder(); } public static Builder newBuilder() {
@Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) return; String hfile = storeFile.getName(); snapshotFiles.add(hfile); if (storeFile.hasReference()) { // Nothing to do here, we have already the reference embedded } else { verifyNonEmptyFile(new Path(exportedArchive, new Path(FSUtils.getTableDir(new Path("./"), tableName), new Path(regionInfo.getEncodedName(), new Path(family, hfile))))); } }
public void writeTo(org.apache.hbase.thirdparty.com.google.protobuf.CodedOutputStream output) throws java.io.IOException { if (((bitField0_ & 0x00000001) == 0x00000001)) { org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessageV3.writeString(output, 1, name_); } if (((bitField0_ & 0x00000002) == 0x00000002)) { output.writeMessage(2, getReference()); } if (((bitField0_ & 0x00000004) == 0x00000004)) { output.writeUInt64(3, fileSize_); } unknownFields.writeTo(output); }
/** * <code>required string name = 1;</code> */ public Builder clearName() { bitField0_ = (bitField0_ & ~0x00000001); name_ = getDefaultInstance().getName(); onChanged(); return this; } /**
@Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { String region = regionInfo.getEncodedName(); String hfile = storeFile.getName(); HFileLink link = HFileLink.build(conf, table, region, family, hfile); if (corruptedFiles.size() % 2 == 0) { fs.delete(link.getAvailablePath(fs), true); corruptedFiles.add(hfile); } } });
List<SnapshotRegionManifest.StoreFile> hfilesToAdd = new ArrayList<>(); for (SnapshotRegionManifest.StoreFile storeFile: snapshotFamilyFiles) { if (familyFiles.contains(storeFile.getName())) { familyFiles.remove(storeFile.getName()); } else { LOG.debug("Adding HFileLink " + storeFile.getName() + " to region=" + regionInfo.getEncodedName() + " table=" + tableName); restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName); restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
final Map<Path, Integer> filesMap) throws IOException { HFileLink link = HFileLink.build(conf, snapshotTable, region.getEncodedName(), family, storeFile.getName()); boolean isCorrupted = false; boolean inArchive = false; hfilesCount.incrementAndGet(); isCorrupted = (storeFile.hasFileSize() && storeFile.getFileSize() != size); if (isCorrupted) hfilesCorrupted.incrementAndGet(); } catch (FileNotFoundException e) {