@Override public long scanOp() throws IOException { // Edit logs of this age don't have any length prefix, so we just have // to read the entire Op. FSEditLogOp op = decodeOp(); return op == null ? HdfsServerConstants.INVALID_TXID : op.getTransactionId(); } }
@Override public long scanOp() throws IOException { if (!NameNodeLayoutVersion.supports( LayoutVersion.Feature.STORED_TXIDS, logVersion)) { throw new IOException("Can't scan a pre-transactional edit log."); } FSEditLogOp op = decodeOp(); return op == null ? HdfsServerConstants.INVALID_TXID : op.getTransactionId(); } }
/** * Skip edit log operations up to a given transaction ID, or until the * end of the edit log is reached. * * After this function returns, the next call to readOp will return either * end-of-file (null) or a transaction with a txid equal to or higher than * the one we asked for. * * @param txid The transaction ID to read up until. * @return Returns true if we found a transaction ID greater than * or equal to 'txid' in the log. */ public boolean skipUntil(long txid) throws IOException { while (true) { FSEditLogOp op = readOp(); if (op == null) { return false; } if (op.getTransactionId() >= txid) { cachedOp = op; return true; } } }
if (syncTxid > 0 && op.getTransactionId() > syncTxid) { breakOuter = true; break; totalEvents += eventBatch.getEvents().length; if (op.getTransactionId() > maxSeenTxid) { maxSeenTxid = op.getTransactionId(); firstSeenTxid = op.getTransactionId(); op.getTransactionId() == syncTxid)) {
op = reader.readOp(skipBrokenEdits); if ((op != null) && (op.hasTransactionId())) { long txId = op.getTransactionId(); if ((txId >= lastTxId) && (lastTxId != HdfsServerConstants.INVALID_TXID)) {
nextTxId = op.getTransactionId(); if (nextTxId <= 0) { nextTxId = 1;
in.getPosition(); if (op.hasTransactionId()) { if (op.getTransactionId() > expectedTxId) { MetaRecoveryContext.editLogLoaderPrompt("There appears " + "to be a gap in the edit log. We expected txid " + expectedTxId + ", but got txid " + op.getTransactionId() + ".", recovery, "ignoring missing " + " transaction IDs"); } else if (op.getTransactionId() < expectedTxId) { MetaRecoveryContext.editLogLoaderPrompt("There appears " + "to be an out-of-order edit in the edit log. We " + "expected txid " + expectedTxId + ", but got txid " + op.getTransactionId() + ".", recovery, "skipping the out-of-order edit"); continue; lastAppliedTxId = op.getTransactionId(); expectedTxId = lastAppliedTxId + 1; } else {
if (fixTxIds) { if (nextTxId <= 0) { nextTxId = op.getTransactionId(); if (nextTxId <= 0) { nextTxId = 1;
prevTxId = op.getTransactionId(); return op; } catch (IOException e) {
/** * Skip edit log operations up to a given transaction ID, or until the * end of the edit log is reached. * * After this function returns, the next call to readOp will return either * end-of-file (null) or a transaction with a txid equal to or higher than * the one we asked for. * * @param txid The transaction ID to read up until. * @return Returns true if we found a transaction ID greater than * or equal to 'txid' in the log. */ public boolean skipUntil(long txid) throws IOException { while (true) { FSEditLogOp op = readOp(); if (op == null) { return false; } if (op.getTransactionId() >= txid) { cachedOp = op; return true; } } }
/** * Skip edit log operations up to a given transaction ID, or until the * end of the edit log is reached. * * After this function returns, the next call to readOp will return either * end-of-file (null) or a transaction with a txid equal to or higher than * the one we asked for. * * @param txid The transaction ID to read up until. * @return Returns true if we found a transaction ID greater than * or equal to 'txid' in the log. */ public boolean skipUntil(long txid) throws IOException { while (true) { FSEditLogOp op = readOp(); if (op == null) { return false; } if (op.getTransactionId() >= txid) { cachedOp = op; return true; } } }
static EditLogValidation scanEditLog(EditLogInputStream in) { long lastPos = 0; long lastTxId = HdfsConstants.INVALID_TXID; long numValid = 0; FSEditLogOp op = null; while (true) { lastPos = in.getPosition(); try { if ((op = in.readOp()) == null) { // TODO break; } } catch (Throwable t) { FSImage.LOG.warn("Caught exception after reading " + numValid + " ops from " + in + " while determining its valid length." + "Position was " + lastPos, t); in.resync(); FSImage.LOG.warn("After resync, position is " + in.getPosition()); continue; } if (lastTxId == HdfsConstants.INVALID_TXID || op.getTransactionId() > lastTxId) { lastTxId = op.getTransactionId(); } numValid++; } return new EditLogValidation(lastPos, lastTxId, false); }
|| op.getTransactionId() > lastTxId) { lastTxId = op.getTransactionId();
private static long readAllEdits(Collection<EditLogInputStream> streams, long startTxId) throws IOException { FSEditLogOp op; long nextTxId = startTxId; long numTx = 0; for (EditLogInputStream s : streams) { while (true) { op = s.readOp(); if (op == null) break; if (op.getTransactionId() != nextTxId) { throw new IOException("out of order transaction ID! expected " + nextTxId + " but got " + op.getTransactionId() + " when " + "reading " + s.getName()); } numTx++; nextTxId = op.getTransactionId() + 1; } } return numTx; }
} else { FSEditLogOp op = decodeOp(); return op == null ? HdfsConstants.INVALID_TXID : op.getTransactionId();
static EditLogValidation scanEditLog(EditLogInputStream in) { long lastPos = 0; long lastTxId = HdfsConstants.INVALID_TXID; long numValid = 0; FSEditLogOp op = null; while (true) { lastPos = in.getPosition(); try { if ((op = in.readOp()) == null) { // TODO break; } } catch (Throwable t) { FSImage.LOG.warn("Caught exception after reading " + numValid + " ops from " + in + " while determining its valid length." + "Position was " + lastPos, t); in.resync(); FSImage.LOG.warn("After resync, position is " + in.getPosition()); continue; } if (lastTxId == HdfsConstants.INVALID_TXID || op.getTransactionId() > lastTxId) { lastTxId = op.getTransactionId(); } numValid++; } return new EditLogValidation(lastPos, lastTxId, false); }
|| op.getTransactionId() > lastTxId) { lastTxId = op.getTransactionId();
/** * Make sure that we starting reading the correct op when we request a stream * with a txid in the middle of an edit log file. */ @Test public void testReadFromMiddleOfEditLog() throws CorruptionException, IOException { File f = new File(TestEditLog.TEST_DIR + "/readfrommiddleofeditlog"); NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10); StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next(); FileJournalManager jm = new FileJournalManager(conf, sd, storage); EditLogInputStream elis = getJournalInputStream(jm, 5, true); try { FSEditLogOp op = elis.readOp(); assertEquals("read unexpected op", op.getTransactionId(), 5); } finally { IOUtils.cleanup(LOG, elis); } }
assertEquals(expected, op.getTransactionId());
/** * Make sure that in-progress streams aren't counted if we don't ask for * them. */ @Test public void testExcludeInProgressStreams() throws CorruptionException, IOException { File f = new File(TestEditLog.TEST_DIR + "/excludeinprogressstreams"); // Don't close the edit log once the files have been set up. NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10, false); StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next(); FileJournalManager jm = new FileJournalManager(conf, sd, storage); // If we exclude the in-progess stream, we should only have 100 tx. assertEquals(100, getNumberOfTransactions(jm, 1, false, false)); EditLogInputStream elis = getJournalInputStream(jm, 90, false); try { FSEditLogOp lastReadOp = null; while ((lastReadOp = elis.readOp()) != null) { assertTrue(lastReadOp.getTransactionId() <= 100); } } finally { IOUtils.cleanup(LOG, elis); } }