static void rollForwardByApplyingLogs( RemoteEditLogManifest manifest, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); List<EditLogInputStream> editsStreams = Lists.newArrayList(); for (RemoteEditLog log : manifest.getLogs()) { if (log.getEndTxId() > dstImage.getLastAppliedTxId()) { File f = dstStorage.findFinalizedEditsFile( log.getStartTxId(), log.getEndTxId()); editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), log.getEndTxId(), true)); } } LOG.info("Checkpointer about to load edits from " + editsStreams.size() + " stream(s)."); dstImage.loadEdits(editsStreams, dstNamesystem); } }
public long loadEdits(Iterable<EditLogInputStream> editStreams, FSNamesystem target, long maxTxnsToRead, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException { LOG.debug("About to load edits:\n " + Joiner.on("\n ").join(editStreams)); StartupProgress prog = NameNode.getStartupProgress(); prog.beginPhase(Phase.LOADING_EDITS); long prevLastAppliedTxId = lastAppliedTxId; try { FSEditLogLoader loader = new FSEditLogLoader(target, lastAppliedTxId); // Load latest edits for (EditLogInputStream editIn : editStreams) { LOG.info("Reading " + editIn + " expecting start txid #" + (lastAppliedTxId + 1)); try { loader.loadFSEdits(editIn, lastAppliedTxId + 1, maxTxnsToRead, startOpt, recovery); } finally { // Update lastAppliedTxId even in case of error, since some ops may // have been successfully applied before the error. lastAppliedTxId = loader.getLastAppliedTxId(); } // If we are in recovery mode, we may have skipped over some txids. if (editIn.getLastTxId() != HdfsServerConstants.INVALID_TXID && recovery != null) { lastAppliedTxId = editIn.getLastTxId(); } }
editsLoaded = image.loadEdits( streams, namesystem, maxTxnsPerLock, null, null); } catch (EditLogInputException elie) {
long txnsAdvanced = loadEdits(editStreams, target, Long.MAX_VALUE, startOpt, recovery); needToSave |= needsResaveBasedOnStaleCheckpoint(imageFile.getFile(),
/** * Load the specified list of edit files into the image. */ public long loadEdits(Iterable<EditLogInputStream> editStreams, FSNamesystem target) throws IOException { return loadEdits(editStreams, target, null, null); }
/** * Load the specified list of edit files into the image. */ public long loadEdits(Iterable<EditLogInputStream> editStreams, FSNamesystem target) throws IOException { return loadEdits(editStreams, target, null, null); }
editsLoaded = image.loadEdits(streams, namesystem); } catch (EditLogInputException elie) { editsLoaded = elie.getNumEditsLoaded();
static void rollForwardByApplyingLogs( RemoteEditLogManifest manifest, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); List<EditLogInputStream> editsStreams = Lists.newArrayList(); for (RemoteEditLog log : manifest.getLogs()) { if (log.getEndTxId() > dstImage.getLastAppliedTxId()) { File f = dstStorage.findFinalizedEditsFile( log.getStartTxId(), log.getEndTxId()); editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), log.getEndTxId(), true)); } } LOG.info("Checkpointer about to load edits from " + editsStreams.size() + " stream(s)."); dstImage.loadEdits(editsStreams, dstNamesystem); } }
static void rollForwardByApplyingLogs( RemoteEditLogManifest manifest, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); List<EditLogInputStream> editsStreams = Lists.newArrayList(); for (RemoteEditLog log : manifest.getLogs()) { if (log.getEndTxId() > dstImage.getLastAppliedTxId()) { File f = dstStorage.findFinalizedEditsFile( log.getStartTxId(), log.getEndTxId()); editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), log.getEndTxId(), true)); } } LOG.info("Checkpointer about to load edits from " + editsStreams.size() + " stream(s)."); dstImage.loadEdits(editsStreams, dstNamesystem); } }
long txnsAdvanced = loadEdits(editStreams, target, startOpt, recovery); needToSave |= needsResaveBasedOnStaleCheckpoint(imageFile.getFile(), txnsAdvanced);
long txnsAdvanced = loadEdits(editStreams, target, startOpt, recovery); needToSave |= needsResaveBasedOnStaleCheckpoint(imageFile.getFile(), txnsAdvanced);
editsLoaded = image.loadEdits(streams, namesystem); } catch (EditLogInputException elie) { editsLoaded = elie.getNumEditsLoaded();