/** * Construct a checkpoint image. * @param conf Node configuration. * @param imageDirs URIs of storage for image. * @param editsDirs URIs of storage for edit logs. * @throws IOException If storage cannot be access. */ CheckpointStorage(Configuration conf, Collection<URI> imageDirs, List<URI> editsDirs) throws IOException { super(conf, imageDirs, editsDirs); // the 2NN never writes edits -- it only downloads them. So // we shouldn't have any editLog instance. Setting to null // makes sure we don't accidentally depend on it. editLog = null; mergeErrorCount = 0; // Replace the archival manager with one that can actually work on the // 2NN's edits storage. this.archivalManager = new NNStorageRetentionManager(conf, storage, new CheckpointLogPurger(storage)); }
/** * Purge any files in the storage directories that are no longer * necessary. */ void purgeOldStorage(NameNodeFile nnf) { try { archivalManager.purgeOldStorage(nnf); } catch (Exception e) { LOG.warn("Unable to purge old storage " + nnf.getName(), e); } }
storage.inspectStorageDirs(inspector); long minImageTxId = getImageTxIdToRetain(inspector); purgeCheckpointsOlderThan(inspector, minImageTxId);
/** rollback for rolling upgrade. */ private void rollingRollback(long discardSegmentTxId, long ckptId) throws IOException { // discard discard unnecessary editlog segments starting from the given id this.editLog.discardSegments(discardSegmentTxId); // rename the special checkpoint renameCheckpoint(ckptId, NameNodeFile.IMAGE_ROLLBACK, NameNodeFile.IMAGE, true); // purge all the checkpoints after the marker archivalManager.purgeCheckpoinsAfter(NameNodeFile.IMAGE, ckptId); // HDFS-7939: purge all old fsimage_rollback_* archivalManager.purgeCheckpoints(NameNodeFile.IMAGE_ROLLBACK); String nameserviceId = DFSUtil.getNamenodeNameServiceId(conf); if (HAUtil.isHAEnabled(conf, nameserviceId)) { // close the editlog since it is currently open for write this.editLog.close(); // reopen the editlog for read this.editLog.initSharedJournalsForRead(); } }
new NNStorageRetentionManager(conf, tc.mockStorage(), tc.mockEditLog(mockPurger), mockPurger) .purgeOldStorage(NameNodeFile.IMAGE);
void purgeCheckpoints(NameNodeFile nnf) throws IOException { purgeCheckpoinsAfter(nnf, -1); }
/** * Save FSimage in the legacy format. This is not for NN consumption, * but for tools like OIV. */ public void saveLegacyOIVImage(FSNamesystem source, String targetDir, Canceler canceler) throws IOException { FSImageCompression compression = FSImageCompression.createCompression(conf); long txid = getCorrectLastAppliedOrWrittenTxId(); SaveNamespaceContext ctx = new SaveNamespaceContext(source, txid, canceler); FSImageFormat.Saver saver = new FSImageFormat.Saver(ctx); String imageFileName = NNStorage.getLegacyOIVImageFileName(txid); File imageFile = new File(targetDir, imageFileName); saver.save(imageFile, compression); archivalManager.purgeOldLegacyOIVImages(targetDir, txid); }
archivalManager.purgeCheckpoints(NameNodeFile.IMAGE_NEW);
void purgeCheckpoints(NameNodeFile nnf) throws IOException { purgeCheckpoinsAfter(nnf, -1); }
/** * Save FSimage in the legacy format. This is not for NN consumption, * but for tools like OIV. */ public void saveLegacyOIVImage(FSNamesystem source, String targetDir, Canceler canceler) throws IOException { FSImageCompression compression = FSImageCompression.createCompression(conf); long txid = getCorrectLastAppliedOrWrittenTxId(); SaveNamespaceContext ctx = new SaveNamespaceContext(source, txid, canceler); FSImageFormat.Saver saver = new FSImageFormat.Saver(ctx); String imageFileName = NNStorage.getLegacyOIVImageFileName(txid); File imageFile = new File(targetDir, imageFileName); saver.save(imageFile, compression); archivalManager.purgeOldLegacyOIVImages(targetDir, txid); }
storage.inspectStorageDirs(inspector); long minImageTxId = getImageTxIdToRetain(inspector); purgeCheckpointsOlderThan(inspector, minImageTxId);
/** * Construct the FSImage. Set the default checkpoint directories. * * Setup storage and initialize the edit log. * * @param conf Configuration * @param imageDirs Directories the image can be stored in. * @param editsDirs Directories the editlog can be stored in. * @throws IOException if directories are invalid. */ protected FSImage(Configuration conf, Collection<URI> imageDirs, List<URI> editsDirs) throws IOException { this.conf = conf; storage = new NNStorage(conf, imageDirs, editsDirs); if(conf.getBoolean(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_RESTORE_KEY, DFSConfigKeys.DFS_NAMENODE_NAME_DIR_RESTORE_DEFAULT)) { storage.setRestoreFailedStorage(true); } this.editLog = FSEditLog.newInstance(conf, storage, editsDirs); archivalManager = new NNStorageRetentionManager(conf, storage, editLog); }
/** * Purge any files in the storage directories that are no longer * necessary. */ void purgeOldStorage(NameNodeFile nnf) { try { archivalManager.purgeOldStorage(nnf); } catch (Exception e) { LOG.warn("Unable to purge old storage " + nnf.getName(), e); } }
void purgeCheckpoints(NameNodeFile nnf) throws IOException { purgeCheckpoinsAfter(nnf, -1); }
/** * Save FSimage in the legacy format. This is not for NN consumption, * but for tools like OIV. */ public void saveLegacyOIVImage(FSNamesystem source, String targetDir, Canceler canceler) throws IOException { FSImageCompression compression = FSImageCompression.createCompression(conf); long txid = getCorrectLastAppliedOrWrittenTxId(); SaveNamespaceContext ctx = new SaveNamespaceContext(source, txid, canceler); FSImageFormat.Saver saver = new FSImageFormat.Saver(ctx); String imageFileName = NNStorage.getLegacyOIVImageFileName(txid); File imageFile = new File(targetDir, imageFileName); saver.save(imageFile, compression); archivalManager.purgeOldLegacyOIVImages(targetDir, txid); }
storage.inspectStorageDirs(inspector); long minImageTxId = getImageTxIdToRetain(inspector); purgeCheckpointsOlderThan(inspector, minImageTxId);
/** * Construct a checkpoint image. * @param conf Node configuration. * @param imageDirs URIs of storage for image. * @param editsDirs URIs of storage for edit logs. * @throws IOException If storage cannot be access. */ CheckpointStorage(Configuration conf, Collection<URI> imageDirs, List<URI> editsDirs) throws IOException { super(conf, imageDirs, editsDirs); // the 2NN never writes edits -- it only downloads them. So // we shouldn't have any editLog instance. Setting to null // makes sure we don't accidentally depend on it. editLog = null; mergeErrorCount = 0; // Replace the archival manager with one that can actually work on the // 2NN's edits storage. this.archivalManager = new NNStorageRetentionManager(conf, storage, new CheckpointLogPurger(storage)); }
/** * Purge any files in the storage directories that are no longer * necessary. */ void purgeOldStorage(NameNodeFile nnf) { try { archivalManager.purgeOldStorage(nnf); } catch (Exception e) { LOG.warn("Unable to purge old storage " + nnf.getName(), e); } }
/** rollback for rolling upgrade. */ private void rollingRollback(long discardSegmentTxId, long ckptId) throws IOException { // discard discard unnecessary editlog segments starting from the given id this.editLog.discardSegments(discardSegmentTxId); // rename the special checkpoint renameCheckpoint(ckptId, NameNodeFile.IMAGE_ROLLBACK, NameNodeFile.IMAGE, true); // purge all the checkpoints after the marker archivalManager.purgeCheckpoinsAfter(NameNodeFile.IMAGE, ckptId); String nameserviceId = DFSUtil.getNamenodeNameServiceId(conf); if (HAUtil.isHAEnabled(conf, nameserviceId)) { // close the editlog since it is currently open for write this.editLog.close(); // reopen the editlog for read this.editLog.initSharedJournalsForRead(); } }
/** * Construct a checkpoint image. * @param conf Node configuration. * @param imageDirs URIs of storage for image. * @param editsDirs URIs of storage for edit logs. * @throws IOException If storage cannot be access. */ CheckpointStorage(Configuration conf, Collection<URI> imageDirs, List<URI> editsDirs) throws IOException { super(conf, imageDirs, editsDirs); // the 2NN never writes edits -- it only downloads them. So // we shouldn't have any editLog instance. Setting to null // makes sure we don't accidentally depend on it. editLog = null; mergeErrorCount = 0; // Replace the archival manager with one that can actually work on the // 2NN's edits storage. this.archivalManager = new NNStorageRetentionManager(conf, storage, new CheckpointLogPurger(storage)); }