/** * Save the contents of the FS image to the file. */ void saveFSImage(SaveNamespaceContext context, StorageDirectory sd, NameNodeFile dstType) throws IOException { long txid = context.getTxId(); File newFile = NNStorage.getStorageFile(sd, NameNodeFile.IMAGE_NEW, txid); File dstFile = NNStorage.getStorageFile(sd, dstType, txid); FSImageFormatProtobuf.Saver saver = new FSImageFormatProtobuf.Saver(context); FSImageCompression compression = FSImageCompression.createCompression(conf); long numErrors = saver.save(newFile, compression); if (numErrors > 0) { // The image is likely corrupted. LOG.error("Detected " + numErrors + " errors while saving FsImage " + dstFile); exitAfterSave.set(true); } MD5FileUtils.saveMD5File(dstFile, saver.getSavedDigest()); storage.setMostRecentCheckpointInfo(txid, Time.now()); }
Iterator<INodeWithAdditionalFields> iter = dir.getINodeMap() .getMapIterator(); final ArrayList<INodeReference> refList = parent.getSaverContext() .getRefList(); int i = 0; parent.commitSection(summary, FSImageFormatProtobuf.SectionName.INODE_DIR);
saveNameSystemSection(b); saveErasureCodingSection(b); prog.endStep(Phase.SAVING_CHECKPOINT, step); prog.beginStep(Phase.SAVING_CHECKPOINT, step); long numErrors = saveInodes(b); numErrors += saveSnapshots(b); prog.endStep(Phase.SAVING_CHECKPOINT, step); saveSecretManagerSection(b); prog.endStep(Phase.SAVING_CHECKPOINT, step); saveCacheManagerSection(b); prog.endStep(Phase.SAVING_CHECKPOINT, step); saveStringTableSection(b); flushSectionOutputStream(); saveFileSummary(underlyingOutputStream, summary); underlyingOutputStream.close(); savedDigest = new MD5Hash(digester.digest());
db.setName(ByteString.copyFrom(copy.getLocalNameBytes())) .setSnapshotCopy( buildINodeDirectory(copy, parent.getSaverContext()));
db.setName(ByteString.copyFrom(copy.getLocalNameBytes())) .setSnapshotCopy( buildINodeDirectory(copy, parent.getSaverContext()));
saveNameSystemSection(b); saveInodes(b); saveSnapshots(b); prog.endStep(Phase.SAVING_CHECKPOINT, step); saveSecretManagerSection(b); prog.endStep(Phase.SAVING_CHECKPOINT, step); saveCacheManagerSection(b); prog.endStep(Phase.SAVING_CHECKPOINT, step); saveStringTableSection(b); flushSectionOutputStream(); saveFileSummary(underlyingOutputStream, summary); underlyingOutputStream.close(); savedDigest = new MD5Hash(digester.digest());
void serializeINodeDirectorySection(OutputStream out) throws IOException { Iterator<INodeWithAdditionalFields> iter = fsn.getFSDirectory() .getINodeMap().getMapIterator(); final ArrayList<INodeReference> refList = parent.getSaverContext() .getRefList(); int i = 0; parent.commitSection(summary, FSImageFormatProtobuf.SectionName.INODE_DIR);
saveNameSystemSection(b); saveInodes(b); saveSnapshots(b); prog.endStep(Phase.SAVING_CHECKPOINT, step); saveSecretManagerSection(b); prog.endStep(Phase.SAVING_CHECKPOINT, step); saveCacheManagerSection(b); prog.endStep(Phase.SAVING_CHECKPOINT, step); saveStringTableSection(b); flushSectionOutputStream(); saveFileSummary(underlyingOutputStream, summary); underlyingOutputStream.close(); savedDigest = new MD5Hash(digester.digest());
.newBuilder().setSnapshotId(s.getId()); INodeSection.INodeDirectory.Builder db = buildINodeDirectory(sroot, parent.getSaverContext()); INodeSection.INode r = INodeSection.INode.newBuilder() .setId(sroot.getId()) parent.commitSection(headers, FSImageFormatProtobuf.SectionName.SNAPSHOT);
.newBuilder().setSnapshotId(s.getId()); INodeSection.INodeDirectory.Builder db = buildINodeDirectory(sroot, parent.getSaverContext()); INodeSection.INode r = INodeSection.INode.newBuilder() .setId(sroot.getId()) parent.commitSection(headers, FSImageFormatProtobuf.SectionName.SNAPSHOT);
void serializeINodeDirectorySection(OutputStream out) throws IOException { Iterator<INodeWithAdditionalFields> iter = fsn.getFSDirectory() .getINodeMap().getMapIterator(); final ArrayList<INodeReference> refList = parent.getSaverContext() .getRefList(); int i = 0; parent.commitSection(summary, FSImageFormatProtobuf.SectionName.INODE_DIR);
private void serializeFileDiffList(INodeFile file, OutputStream out) throws IOException { FileWithSnapshotFeature sf = file.getFileWithSnapshotFeature(); if (sf != null) { List<FileDiff> diffList = sf.getDiffs().asList(); SnapshotDiffSection.DiffEntry entry = SnapshotDiffSection.DiffEntry .newBuilder().setInodeId(file.getId()).setType(Type.FILEDIFF) .setNumOfDiff(diffList.size()).build(); entry.writeDelimitedTo(out); for (int i = diffList.size() - 1; i >= 0; i--) { FileDiff diff = diffList.get(i); SnapshotDiffSection.FileDiff.Builder fb = SnapshotDiffSection.FileDiff .newBuilder().setSnapshotId(diff.getSnapshotId()) .setFileSize(diff.getFileSize()); if(diff.getBlocks() != null) { for(Block block : diff.getBlocks()) { fb.addBlocks(PBHelper.convert(block)); } } INodeFileAttributes copy = diff.snapshotINode; if (copy != null) { fb.setName(ByteString.copyFrom(copy.getLocalNameBytes())) .setSnapshotCopy(buildINodeFile(copy, parent.getSaverContext())); } fb.build().writeDelimitedTo(out); } } }
/** * save all the snapshot diff to fsimage */ public void serializeSnapshotDiffSection(OutputStream out) throws IOException { INodeMap inodesMap = fsn.getFSDirectory().getINodeMap(); final List<INodeReference> refList = parent.getSaverContext() .getRefList(); int i = 0; Iterator<INodeWithAdditionalFields> iter = inodesMap.getMapIterator(); while (iter.hasNext()) { INodeWithAdditionalFields inode = iter.next(); if (inode.isFile()) { serializeFileDiffList(inode.asFile(), out); } else if (inode.isDirectory()) { serializeDirDiffList(inode.asDirectory(), refList, out); } ++i; if (i % FSImageFormatProtobuf.Saver.CHECK_CANCEL_INTERVAL == 0) { context.checkCancelled(); } } parent.commitSection(headers, FSImageFormatProtobuf.SectionName.SNAPSHOT_DIFF); }
private void serializeFileDiffList(INodeFile file, OutputStream out) throws IOException { FileWithSnapshotFeature sf = file.getFileWithSnapshotFeature(); if (sf != null) { List<FileDiff> diffList = sf.getDiffs().asList(); SnapshotDiffSection.DiffEntry entry = SnapshotDiffSection.DiffEntry .newBuilder().setInodeId(file.getId()).setType(Type.FILEDIFF) .setNumOfDiff(diffList.size()).build(); entry.writeDelimitedTo(out); for (int i = diffList.size() - 1; i >= 0; i--) { FileDiff diff = diffList.get(i); SnapshotDiffSection.FileDiff.Builder fb = SnapshotDiffSection.FileDiff .newBuilder().setSnapshotId(diff.getSnapshotId()) .setFileSize(diff.getFileSize()); if(diff.getBlocks() != null) { for(Block block : diff.getBlocks()) { fb.addBlocks(PBHelper.convert(block)); } } INodeFileAttributes copy = diff.snapshotINode; if (copy != null) { fb.setName(ByteString.copyFrom(copy.getLocalNameBytes())) .setSnapshotCopy(buildINodeFile(copy, parent.getSaverContext())); } fb.build().writeDelimitedTo(out); } } }
private void saveNameSystemSection(FileSummary.Builder summary) throws IOException { final FSNamesystem fsn = context.getSourceNamesystem(); OutputStream out = sectionOutputStream; BlockIdManager blockIdManager = fsn.getBlockIdManager(); NameSystemSection.Builder b = NameSystemSection.newBuilder() .setGenstampV1(blockIdManager.getGenerationStampV1()) .setGenstampV1Limit(blockIdManager.getGenerationStampV1Limit()) .setGenstampV2(blockIdManager.getGenerationStampV2()) .setLastAllocatedBlockId(blockIdManager.getLastAllocatedBlockId()) .setTransactionId(context.getTxId()); // We use the non-locked version of getNamespaceInfo here since // the coordinating thread of saveNamespace already has read-locked // the namespace for us. If we attempt to take another readlock // from the actual saver thread, there's a potential of a // fairness-related deadlock. See the comments on HDFS-2223. b.setNamespaceId(fsn.unprotectedGetNamespaceInfo().getNamespaceID()); if (fsn.isRollingUpgrade()) { b.setRollingUpgradeStartTime(fsn.getRollingUpgradeInfo().getStartTime()); } NameSystemSection s = b.build(); s.writeDelimitedTo(out); commitSection(summary, SectionName.NS_INFO); }
/** * save all the snapshot diff to fsimage */ public void serializeSnapshotDiffSection(OutputStream out) throws IOException { INodeMap inodesMap = fsn.getFSDirectory().getINodeMap(); final List<INodeReference> refList = parent.getSaverContext() .getRefList(); int i = 0; Iterator<INodeWithAdditionalFields> iter = inodesMap.getMapIterator(); while (iter.hasNext()) { INodeWithAdditionalFields inode = iter.next(); if (inode.isFile()) { serializeFileDiffList(inode.asFile(), out); } else if (inode.isDirectory()) { serializeDirDiffList(inode.asDirectory(), refList, out); } ++i; if (i % FSImageFormatProtobuf.Saver.CHECK_CANCEL_INTERVAL == 0) { context.checkCancelled(); } } parent.commitSection(headers, FSImageFormatProtobuf.SectionName.SNAPSHOT_DIFF); }
private void saveNameSystemSection(FileSummary.Builder summary) throws IOException { final FSNamesystem fsn = context.getSourceNamesystem(); OutputStream out = sectionOutputStream; BlockIdManager blockIdManager = fsn.getBlockIdManager(); NameSystemSection.Builder b = NameSystemSection.newBuilder() .setGenstampV1(blockIdManager.getGenerationStampV1()) .setGenstampV1Limit(blockIdManager.getGenerationStampV1Limit()) .setGenstampV2(blockIdManager.getGenerationStampV2()) .setLastAllocatedBlockId(blockIdManager.getLastAllocatedBlockId()) .setTransactionId(context.getTxId()); // We use the non-locked version of getNamespaceInfo here since // the coordinating thread of saveNamespace already has read-locked // the namespace for us. If we attempt to take another readlock // from the actual saver thread, there's a potential of a // fairness-related deadlock. See the comments on HDFS-2223. b.setNamespaceId(fsn.unprotectedGetNamespaceInfo().getNamespaceID()); if (fsn.isRollingUpgrade()) { b.setRollingUpgradeStartTime(fsn.getRollingUpgradeInfo().getStartTime()); } NameSystemSection s = b.build(); s.writeDelimitedTo(out); commitSection(summary, SectionName.NS_INFO); }
private void save(OutputStream out, INodeFile n) throws IOException { INodeSection.INodeFile.Builder b = buildINodeFile(n, parent.getSaverContext()); if (n.getBlocks() != null) { for (Block block : n.getBlocks()) { b.addBlocks(PBHelper.convert(block)); } } FileUnderConstructionFeature uc = n.getFileUnderConstructionFeature(); if (uc != null) { INodeSection.FileUnderConstructionFeature f = INodeSection.FileUnderConstructionFeature .newBuilder().setClientName(uc.getClientName()) .setClientMachine(uc.getClientMachine()).build(); b.setFileUC(f); } INodeSection.INode r = buildINodeCommon(n) .setType(INodeSection.INode.Type.FILE).setFile(b).build(); r.writeDelimitedTo(out); }
private void save(OutputStream out, INodeFile n) throws IOException { INodeSection.INodeFile.Builder b = buildINodeFile(n, parent.getSaverContext()); if (n.getBlocks() != null) { for (Block block : n.getBlocks()) { b.addBlocks(PBHelper.convert(block)); } } FileUnderConstructionFeature uc = n.getFileUnderConstructionFeature(); if (uc != null) { INodeSection.FileUnderConstructionFeature f = INodeSection.FileUnderConstructionFeature .newBuilder().setClientName(uc.getClientName()) .setClientMachine(uc.getClientMachine()).build(); b.setFileUC(f); } INodeSection.INode r = buildINodeCommon(n) .setType(INodeSection.INode.Type.FILE).setFile(b).build(); r.writeDelimitedTo(out); }
void serializeINodeSection(OutputStream out) throws IOException { INodeMap inodesMap = fsn.dir.getINodeMap(); INodeSection.Builder b = INodeSection.newBuilder() .setLastInodeId(fsn.dir.getLastInodeId()).setNumInodes(inodesMap.size()); INodeSection s = b.build(); s.writeDelimitedTo(out); int i = 0; Iterator<INodeWithAdditionalFields> iter = inodesMap.getMapIterator(); while (iter.hasNext()) { INodeWithAdditionalFields n = iter.next(); save(out, n); ++i; if (i % FSImageFormatProtobuf.Saver.CHECK_CANCEL_INTERVAL == 0) { context.checkCancelled(); } } parent.commitSection(summary, FSImageFormatProtobuf.SectionName.INODE); }