FSEditLogLoader loader = new FSEditLogLoader(target, lastAppliedTxId); (lastAppliedTxId + 1)); try { loader.loadFSEdits(editIn, lastAppliedTxId + 1, maxTxnsToRead, startOpt, recovery); } finally { lastAppliedTxId = loader.getLastAppliedTxId();
long numTxns = (lastTxId - expectedStartingTxId) + 1; StartupProgress prog = NameNode.getStartupProgress(); Step step = createStartupProgressStep(in); prog.setTotal(Phase.LOADING_EDITS, step, numTxns); Counter counter = prog.getCounter(Phase.LOADING_EDITS, step); check203UpgradeFailure(in.getVersion(true), e); String errorMessage = formatEditLogReplayError(in, recentOpcodeOffsets, expectedTxId); FSImage.LOG.error(errorMessage, e); if (recovery == null) { + ", numEdits=" + numEdits + ", totalEdits=" + totalEdits); long inodeId = applyEditLogOp(op, fsDir, startOpt, in.getVersion(true), lastInodeId); if (lastInodeId < inodeId) { incrOpCount(op.opCode, opCounts, step, counter); if (op.hasTransactionId()) { lastAppliedTxId = op.getTransactionId(); dumpOpCounts(opCounts); FSImage.LOG.debug("maxTxnsToRead = " + maxTxnsToRead + " actual edits read = " + numEdits);
new FSEditLogLoader(getNamesystem(), lastAppliedTxId); int logVersion = storage.getLayoutVersion(); backupInputStream.setBytes(data, logVersion); long numTxnsAdvanced = logLoader.loadEditRecords( backupInputStream, true, lastAppliedTxId + 1, null, null); if (numTxnsAdvanced != numTxns) { numTxnsAdvanced); lastAppliedTxId = logLoader.getLastAppliedTxId();
/** * Load an edit log, and apply the changes to the in-memory structure * This is where we apply edits that we've been writing to disk all * along. */ long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId, long maxTxnsToRead, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); Step step = createStartupProgressStep(edits); prog.beginStep(Phase.LOADING_EDITS, step); fsNamesys.writeLock(); try { long startTime = monotonicNow(); FSImage.LOG.info("Start loading edits file " + edits.getName() + " maxTxnsToRead = " + maxTxnsToRead); long numEdits = loadEditRecords(edits, false, expectedStartingTxId, maxTxnsToRead, startOpt, recovery); FSImage.LOG.info("Edits file " + edits.getName() + " of size " + edits.length() + " edits # " + numEdits + " loaded in " + (monotonicNow()-startTime)/1000 + " seconds"); return numEdits; } finally { edits.close(); fsNamesys.writeUnlock("loadFSEdits"); prog.endStep(Phase.LOADING_EDITS, step); } }
private void verifyEditLogs(FSNamesystem namesystem, FSImage fsimage) throws IOException { // Verify that we can read in all the transactions that we have written. // If there were any corruptions, it is likely that the reading in // of these transactions will throw an exception. for (Iterator<StorageDirectory> it = fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) { File editFile = FSImage.getImageFile(it.next(), NameNodeFile.EDITS); System.out.println("Verifying file: " + editFile); int numEdits = new FSEditLogLoader(namesystem).loadFSEdits( new EditLogFileInputStream(editFile)); System.out.println("Number of edits: " + numEdits); } }
/** * Load and merge edits from two edits files * * @param sd storage directory * @return number of edits loaded * @throws IOException */ int loadFSEdits(StorageDirectory sd) throws IOException { int numEdits = 0; EditLogFileInputStream edits = new EditLogFileInputStream(getImageFile(sd, NameNodeFile.EDITS)); FSEditLogLoader loader = new FSEditLogLoader(namesystem); numEdits += loader.loadFSEdits(edits, editLog.getCurrentTxId()); edits.close(); File editsNew = getImageFile(sd, NameNodeFile.EDITS_NEW); if (editsNew.exists() && editsNew.length() > 0) { edits = new EditLogFileInputStream(editsNew); numEdits += loader.loadFSEdits(edits, loader.getCurrentTxId()); edits.close(); } editLog.setStartTransactionId(loader.getCurrentTxId()); // update the counts. getFSNamesystem().dir.updateCountForINodeWithQuota(); return numEdits; }
inodeId = getAndUpdateLastInodeId(addCloseOp.inodeId, logVersion, lastInodeId); newFile = FSDirWriteFileOp.addFileForEditLog(fsDir, inodeId, iip.getExistingINodes(), iip.getLastLocalName(), FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy( fsDir.getFSNamesystem(), iip); updateBlocks(fsDir, addCloseOp, iip, newFile, ecPolicy); break; FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy( fsDir.getFSNamesystem(), iip); updateBlocks(fsDir, addCloseOp, iip, file, ecPolicy); FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy( fsDir.getFSNamesystem(), iip); updateBlocks(fsDir, updateOp, iip, oldFile, ecPolicy); FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy( fsDir.getFSNamesystem(), iip); addNewBlock(addBlockOp, oldFile, ecPolicy); break; inodeId = getAndUpdateLastInodeId(mkdirOp.inodeId, logVersion, lastInodeId); FSDirMkdirOp.mkdirForEditLog(fsDir, inodeId, inodeId = getAndUpdateLastInodeId(symlinkOp.inodeId, logVersion, lastInodeId); final String path = renameReservedPathsOnUpgrade(symlinkOp.path,
long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId) throws IOException { return loadFSEdits(edits, expectedStartingTxId, Long.MAX_VALUE, null, null); }
static FSEditLogLoader.EditLogValidation validateEditLog(File file) throws IOException { EditLogFileInputStream in; try { in = new EditLogFileInputStream(file); } catch (LogHeaderCorruptException corrupt) { // If it's missing its header, this is equivalent to no transactions FSImage.LOG.warn("Log at " + file + " has no valid header", corrupt); return new FSEditLogLoader.EditLogValidation(0, HdfsConstants.INVALID_TXID, HdfsConstants.INVALID_TXID); } try { return FSEditLogLoader.validateEditLog(in); } finally { IOUtils.closeStream(in); } }
loadEditRecord(logVersion, in, recentOpcodeOffsets, dumpOpCounts(opCounts);
long loadEditRecords(EditLogInputStream in, boolean closeOnExit, long expectedStartingTxId, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException { return loadEditRecords(in, closeOnExit, expectedStartingTxId, Long.MAX_VALUE, startOpt, recovery); }
incrOpCount(op.opCode, opCounts); switch (op.opCode) { case OP_ADD:
private long testLoad(byte[] data, FSNamesystem namesys) throws IOException { FSEditLogLoader loader = new FSEditLogLoader(namesys, 0); return loader.loadFSEdits(new EditLogByteInputStream(data), 1); }
inodeId = getAndUpdateLastInodeId(addCloseOp.inodeId, logVersion, lastInodeId); newFile = fsDir.addFileForEditLog(inodeId, iip.getExistingINodes(), iip.getLastLocalName(), updateBlocks(fsDir, addCloseOp, iip, newFile); break; updateBlocks(fsDir, addCloseOp, iip, file); INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path); updateBlocks(fsDir, updateOp, iip, oldFile); addNewBlock(fsDir, addBlockOp, oldFile); break; inodeId = getAndUpdateLastInodeId(mkdirOp.inodeId, logVersion, lastInodeId); FSDirMkdirOp.mkdirForEditLog(fsDir, inodeId, inodeId = getAndUpdateLastInodeId(symlinkOp.inodeId, logVersion, lastInodeId); final String path = renameReservedPathsOnUpgrade(symlinkOp.path,
/** * Load an edit log, and apply the changes to the in-memory structure * This is where we apply edits that we've been writing to disk all * along. */ long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); Step step = createStartupProgressStep(edits); prog.beginStep(Phase.LOADING_EDITS, step); fsNamesys.writeLock(); try { long startTime = monotonicNow(); FSImage.LOG.info("Start loading edits file " + edits.getName()); long numEdits = loadEditRecords(edits, false, expectedStartingTxId, startOpt, recovery); FSImage.LOG.info("Edits file " + edits.getName() + " of size " + edits.length() + " edits # " + numEdits + " loaded in " + (monotonicNow()-startTime)/1000 + " seconds"); return numEdits; } finally { edits.close(); fsNamesys.writeUnlock("loadFSEdits"); prog.endStep(Phase.LOADING_EDITS, step); } }
long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId) throws IOException { return loadFSEdits(edits, expectedStartingTxId, null, null); }
static FSEditLogLoader.EditLogValidation validateEditLog(File file) throws IOException { EditLogFileInputStream in; try { in = new EditLogFileInputStream(file); in.getVersion(true); // causes us to read the header } catch (LogHeaderCorruptException e) { // If the header is malformed or the wrong value, this indicates a corruption LOG.warn("Log file " + file + " has no valid header", e); return new FSEditLogLoader.EditLogValidation(0, HdfsConstants.INVALID_TXID, true); } try { return FSEditLogLoader.validateEditLog(in); } finally { IOUtils.closeStream(in); } }
int loadFSEdits(EditLogInputStream edits, boolean closeOnExit) throws IOException { int numEdits = 0; int logVersion = edits.getVersion(); try { numEdits = loadEditRecords(logVersion, edits, false); } finally { if(closeOnExit) { edits.close(); } } if (logVersion != FSConstants.LAYOUT_VERSION) // other version numEdits++; // save this image asap return numEdits; }
new FSEditLogLoader(getNamesystem(), lastAppliedTxId); loader.loadFSEdits(stream, lastAppliedTxId + 1); lastAppliedTxId = loader.getLastAppliedTxId(); assert lastAppliedTxId == getEditLog().getLastWrittenTxId(); } finally {
long numTxns = (lastTxId - expectedStartingTxId) + 1; StartupProgress prog = NameNode.getStartupProgress(); Step step = createStartupProgressStep(in); prog.setTotal(Phase.LOADING_EDITS, step, numTxns); Counter counter = prog.getCounter(Phase.LOADING_EDITS, step); check203UpgradeFailure(in.getVersion(true), e); String errorMessage = formatEditLogReplayError(in, recentOpcodeOffsets, expectedTxId); FSImage.LOG.error(errorMessage, e); if (recovery == null) { + ", numEdits=" + numEdits + ", totalEdits=" + totalEdits); long inodeId = applyEditLogOp(op, fsDir, startOpt, in.getVersion(true), lastInodeId); if (lastInodeId < inodeId) { incrOpCount(op.opCode, opCounts, step, counter); if (op.hasTransactionId()) { lastAppliedTxId = op.getTransactionId(); dumpOpCounts(opCounts);