FSImage.getCheckpointDirs(conf, null); List<URI> checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, null); FSImage ckptImage = new FSImage(conf, checkpointDirs, checkpointEditsDirs); ckptImage.recoverTransitionRead(StartupOption.REGULAR, target, null); } finally { ckptImage.close(); realImage.getStorage().setStorageInfo(ckptImage.getStorage()); realImage.getEditLog().setNextTxId(ckptImage.getEditLog().getLastWrittenTxId()+1); realImage.initEditLog(StartupOption.IMPORT); realImage.getStorage().setBlockPoolID(ckptImage.getBlockPoolID()); saveNamespace(target); updateStorageVersion();
initEditLog(startOpt); LayoutVersion.Feature.TXID_BASED_LAYOUT, getLayoutVersion())) { try { imageFile = imageFiles.get(i); loadFSImageFile(target, recovery, imageFile, startOpt); break; } catch (IllegalReservedPathException ie) { long txnsAdvanced = loadEdits(editStreams, target, Long.MAX_VALUE, startOpt, recovery); needToSave |= needsResaveBasedOnStaleCheckpoint(imageFile.getFile(), txnsAdvanced); } else { rollingRollback(lastAppliedTxId + 1, imageFiles.get(0).getCheckpointTxId()); needToSave = false;
boolean isFormatted = recoverStorageDirs(startOpt, storage, dataDirStates); case UPGRADE: case UPGRADEONLY: doUpgrade(target); return false; // upgrade saved image already case IMPORT: doImportCheckpoint(target); return false; // import checkpoint saved image already case ROLLBACK: return loadFSImage(target, startOpt, recovery);
CheckpointSignature(FSImage fsImage) { super(fsImage.getStorage()); blockpoolID = fsImage.getBlockPoolID(); mostRecentCheckpointTxId = fsImage.getStorage().getMostRecentCheckpointTxId(); curSegmentTxId = fsImage.getEditLog().getCurSegmentTxId(); }
CheckpointSignature rollEditLog(int layoutVersion) throws IOException { getEditLog().rollEditLog(layoutVersion); // Record this log segment ID in all of the storage directories, so // we won't miss this log segment on a restart if the edits directories // go missing. storage.writeTransactionIdFileToStorage(getEditLog().getCurSegmentTxId()); //Update NameDirSize Metric getStorage().updateNameDirSize(); return new CheckpointSignature(this); }
FSImage image = new FSImage(conf); try { image.getStorage().setStorageInfo(storage); image.initEditLog(StartupOption.REGULAR); assert image.getEditLog().isOpenForRead() : "Expected edit log to be open for read"; image.saveDigestAndRenameCheckpointImage(NameNodeFile.IMAGE, imageTxId, hash); throw ioe; } finally { image.close();
setStorageDirectories(dataDirs, editsDirs); boolean isFormatted = false; for (Iterator<StorageDirectory> it = dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); StorageState curState; throw new IOException("NameNode is not formatted."); if (layoutVersion < LAST_PRE_UPGRADE_LAYOUT_VERSION) { checkVersionUpgradable(layoutVersion); verifyDistributedUpgradeProgress(startOpt); dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); StorageState curState = dataDirStates.get(sd); doUpgrade(); return false; // upgrade saved image already case IMPORT: doImportCheckpoint(); return true; case ROLLBACK: doRollback(); break; case REGULAR: return loadFSImage();
fsImage.setStorageDirectories(dataDirs, editsDirs); fsImage.format(); startOpt = StartupOption.REGULAR; fsImage.recoverTransitionRead(dataDirs, editsDirs, startOpt); if (saveNamespace) { fsImage.saveNamespace(true); FSEditLog editLog = fsImage.getEditLog(); assert editLog != null : "editLog must be initialized"; if (!editLog.isOpen()) editLog.open(); if (!saveNamespace) { fsImage.rollEditLog(); fsImage.setCheckpointDirectories(null, null); } catch(IOException e) { fsImage.close(); throw e;
void loadFSImage(Collection<File> dataDirs, Collection<File> editsDirs, StartupOption startOpt) throws IOException { // format before starting up if requested if (startOpt == StartupOption.FORMAT) { fsImage.setStorageDirectories(dataDirs, editsDirs); fsImage.format(); startOpt = StartupOption.REGULAR; } try { if (fsImage.recoverTransitionRead(dataDirs, editsDirs, startOpt)) { fsImage.saveFSImage(); } FSEditLog editLog = fsImage.getEditLog(); assert editLog != null : "editLog must be initialized"; if (!editLog.isOpen()) editLog.open(); fsImage.setCheckpointDirectories(null, null); } catch(IOException e) { fsImage.close(); throw e; } synchronized (this) { this.ready = true; this.notifyAll(); } }
fsImage.format(this, fsImage.getStorage().determineClusterId(), false); = fsImage.recoverTransitionRead(startOpt, this, recovery); if (RollingUpgradeStartupOption.ROLLBACK.matches(startOpt)) { rollingUpgradeInfo = null; + ", isRollingUpgrade=" + isRollingUpgrade() + ")"); if (needToSave) { fsImage.saveNamespace(this); } else { fsImage.openEditLogForWrite(getEffectiveLayoutVersion()); fsImage.close();
FSImage fsImage = new FSImage(conf, nameDirsToFormat, editDirsToFormat); try { FSNamesystem fsn = new FSNamesystem(conf, fsImage); fsImage.getEditLog().initJournalsForWrite(); if (!fsImage.confirmFormat(force, isInteractive)) { return true; // aborted fsImage.format(fsn, clusterId, force); } catch (IOException ioe) { LOG.warn("Encountered exception during format: ", ioe); fsImage.close(); throw ioe;
FSNamesystem.loadFromDisk(getConfigurationWithoutSharedEdits(conf)); existingStorage = fsns.getFSImage().getStorage(); NamespaceInfo nsInfo = existingStorage.getNamespaceInfo(); sharedEditsImage = new FSImage(conf, Lists.<URI>newArrayList(), sharedEditsDirs); sharedEditsImage.getEditLog().initJournalsForWrite(); if (!sharedEditsImage.confirmFormat(force, interactive)) { return true; // abort NNStorage newSharedStorage = sharedEditsImage.getStorage(); sharedEditsImage.getEditLog().formatNonFileJournals(nsInfo, force); fsns.getFSImage().getEditLog().close(); fsns.getFSImage().getEditLog().initJournalsForWrite(); fsns.getFSImage().getEditLog().recoverUnclosedStreams(); if (sharedEditsImage != null) { try { sharedEditsImage.close(); } catch (IOException ioe) { LOG.warn("Could not close sharedEditsImage", ioe);
private void doUpgrade() throws IOException { if(getDistributedUpgradeState()) { this.loadFSImage(); initializeDistributedUpgrade(); return; dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); if (sd.getPreviousDir().exists()) this.loadFSImage(); long oldCTime = this.getCTime(); this.cTime = FSNamesystem.now(); // generate new cTime for the state int oldLV = this.getLayoutVersion(); this.layoutVersion = FSConstants.LAYOUT_VERSION; this.checkpointTime = FSNamesystem.now(); for (Iterator<StorageDirectory> it = dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); LOG.info("Upgrading image directory " + sd.getRoot() + ".\n old LV = " + oldLV + "; old CTime = " + oldCTime + ".\n new LV = " + this.getLayoutVersion() + "; new CTime = " + this.getCTime()); File curDir = sd.getCurrentDir(); File prevDir = sd.getPreviousDir(); rename(curDir, tmpDir);
/** * Load image from a checkpoint directory and save it into the current one. * @throws IOException */ void doImportCheckpoint() throws IOException { FSNamesystem fsNamesys = getFSNamesystem(); FSImage ckptImage = new FSImage(fsNamesys); // replace real image with the checkpoint image FSImage realImage = fsNamesys.getFSImage(); assert realImage == this; fsNamesys.dir.fsImage = ckptImage; // load from the checkpoint dirs try { ckptImage.recoverTransitionRead(checkpointDirs, checkpointEditsDirs, StartupOption.REGULAR); } finally { ckptImage.close(); } // return back the real image realImage.setStorageInfo(ckptImage); fsNamesys.dir.fsImage = realImage; // and save it saveNamespace(false); }
/** * Load image from a checkpoint directory and save it into the current one. * @throws IOException */ void doImportCheckpoint() throws IOException { FSImage ckptImage = new FSImage(); FSNamesystem fsNamesys = FSNamesystem.getFSNamesystem(); // replace real image with the checkpoint image FSImage realImage = fsNamesys.getFSImage(); assert realImage == this; fsNamesys.dir.fsImage = ckptImage; // load from the checkpoint dirs try { ckptImage.recoverTransitionRead(checkpointDirs, checkpointEditsDirs, StartupOption.REGULAR); } finally { ckptImage.close(); } // return back the real image realImage.setStorageInfo(ckptImage); fsNamesys.dir.fsImage = realImage; // and save it saveFSImage(); }
checkpointDirs = FSImage.getCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary"); checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary"); checkpointImage = new CheckpointStorage(conf, checkpointDirs, checkpointEditsDirs);
NNStorage storage = originalImage.getStorage(); when(spyImage).saveFSImage( (SaveNamespaceContext)anyObject(), (StorageDirectory)anyObject(), (NameNodeFile) anyObject()); when(spyImage).saveFSImage( (SaveNamespaceContext)anyObject(), (StorageDirectory)anyObject(), (NameNodeFile) anyObject()); when(spyImage).saveFSImage( (SaveNamespaceContext)anyObject(), (StorageDirectory)anyObject(), (NameNodeFile) anyObject()); originalImage.close(); fsn.close(); fsn = null;
FSImage prevState = new FSImage(); prevState.layoutVersion = FSConstants.LAYOUT_VERSION; for (Iterator<StorageDirectory> it = dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); File prevDir = sd.getPreviousDir(); dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); File prevDir = sd.getPreviousDir(); + ".\n new LV = " + prevState.getLayoutVersion() + "; new CTime = " + prevState.getCTime()); File tmpDir = sd.getRemovedTmp(); assert !tmpDir.exists() : "removed.tmp directory must not exist."; rename(curDir, tmpDir); rename(prevDir, curDir); deleteDir(tmpDir); LOG.info("Rollback of " + sd.getRoot()+ " is complete."); verifyDistributedUpgradeProgress(StartupOption.REGULAR);
originalImage.unlockAll(); FSImage spyImage = spy(originalImage); fsn.dir.fsImage = spyImage; spyImage.setStorageDirectories( FSNamesystem.getNamespaceDirs(conf), FSNamesystem.getNamespaceEditsDirs(conf)); when(spyImage).saveFSImage((File)anyObject()); spyImage.setRestoreFailedStorage(true); spyImage.attemptRestoreRemovedStorage(); fsn.saveNamespace(); checkEditExists(fsn, 1); originalImage.close(); fsn.close(); fsn = null;
int layoutVersion) // active name-node throws IOException { LOG.info("Start checkpoint at txid " + getEditLog().getLastWrittenTxId()); String msg = null; CheckpointSignature sig = rollEditLog(layoutVersion); return new CheckpointCommand(sig, needToReturnImg);