/** * Go through the next operation from the stream storage. * @return the txid of the next operation. */ protected long scanNextOp() throws IOException { FSEditLogOp next = readOp(); return next != null ? next.txid : HdfsServerConstants.INVALID_TXID; }
private static FSEditLogOp readOp(EditLogInputStream elis) throws IOException { try { return elis.readOp(); // we can get the below two exceptions if a segment is deleted // (because we have accumulated too many edits) or (for the local journal/ // no-QJM case only) if a in-progress segment is finalized under us ... // no need to throw an exception back to the client in this case } catch (FileNotFoundException e) { LOG.debug("Tried to read from deleted or moved edit log segment", e); return null; } catch (HttpGetFailedException e) { LOG.debug("Tried to read from deleted edit log segment", e); return null; } }
/** * Skip edit log operations up to a given transaction ID, or until the * end of the edit log is reached. * * After this function returns, the next call to readOp will return either * end-of-file (null) or a transaction with a txid equal to or higher than * the one we asked for. * * @param txid The transaction ID to read up until. * @return Returns true if we found a transaction ID greater than * or equal to 'txid' in the log. */ public boolean skipUntil(long txid) throws IOException { while (true) { FSEditLogOp op = readOp(); if (op == null) { return false; } if (op.getTransactionId() >= txid) { cachedOp = op; return true; } } }
FSEditLogOp op; boolean segmentOpen = false; while ((op = stream.readOp()) != null) { if (LOG.isTraceEnabled()) { LOG.trace("copying op: " + op);
/** * Go through the next operation from the stream storage. * @return the txid of the next operation. */ protected long scanNextOp() throws IOException { FSEditLogOp next = readOp(); return next != null ? next.txid : HdfsConstants.INVALID_TXID; }
while (true) { try { FSEditLogOp op = inputStream.readOp(); if (op == null) break;
/** * Go through the next operation from the stream storage. * @return the txid of the next operation. */ protected long scanNextOp() throws IOException { FSEditLogOp next = readOp(); return next != null ? next.txid : HdfsConstants.INVALID_TXID; }
case OK: try { FSEditLogOp op = streams[curIdx].readOp(); if (op == null) { state = State.EOF;
FSEditLogOp op; try { op = in.readOp(); if (op == null) { break;
private static FSEditLogOp readOp(EditLogInputStream elis) throws IOException { try { return elis.readOp(); // we can get the below two exceptions if a segment is deleted // (because we have accumulated too many edits) or (for the local journal/ // no-QJM case only) if a in-progress segment is finalized under us ... // no need to throw an exception back to the client in this case } catch (FileNotFoundException e) { LOG.debug("Tried to read from deleted or moved edit log segment", e); return null; } catch (TransferFsImage.HttpGetFailedException e) { LOG.debug("Tried to read from deleted edit log segment", e); return null; } }
/** * Skip edit log operations up to a given transaction ID, or until the * end of the edit log is reached. * * After this function returns, the next call to readOp will return either * end-of-file (null) or a transaction with a txid equal to or higher than * the one we asked for. * * @param txid The transaction ID to read up until. * @return Returns true if we found a transaction ID greater than * or equal to 'txid' in the log. */ public boolean skipUntil(long txid) throws IOException { while (true) { FSEditLogOp op = readOp(); if (op == null) { return false; } if (op.getTransactionId() >= txid) { cachedOp = op; return true; } } }
/** * Skip edit log operations up to a given transaction ID, or until the * end of the edit log is reached. * * After this function returns, the next call to readOp will return either * end-of-file (null) or a transaction with a txid equal to or higher than * the one we asked for. * * @param txid The transaction ID to read up until. * @return Returns true if we found a transaction ID greater than * or equal to 'txid' in the log. */ public boolean skipUntil(long txid) throws IOException { while (true) { FSEditLogOp op = readOp(); if (op == null) { return false; } if (op.getTransactionId() >= txid) { cachedOp = op; return true; } } }
private static FSEditLogOp readOp(EditLogInputStream elis) throws IOException { try { return elis.readOp(); // we can get the below two exceptions if a segment is deleted // (because we have accumulated too many edits) or (for the local journal/ // no-QJM case only) if a in-progress segment is finalized under us ... // no need to throw an exception back to the client in this case } catch (FileNotFoundException e) { LOG.debug("Tried to read from deleted or moved edit log segment", e); return null; } catch (TransferFsImage.HttpGetFailedException e) { LOG.debug("Tried to read from deleted edit log segment", e); return null; } }
/** * @see #countEditLogOpTypes(File) */ public static EnumMap<FSEditLogOpCodes, Holder<Integer>> countEditLogOpTypes( EditLogInputStream elis) throws IOException { EnumMap<FSEditLogOpCodes, Holder<Integer>> opCounts = new EnumMap<FSEditLogOpCodes, Holder<Integer>>(FSEditLogOpCodes.class); FSEditLogOp op; while ((op = elis.readOp()) != null) { Holder<Integer> i = opCounts.get(op.opCode); if (i == null) { i = new Holder<Integer>(0); opCounts.put(op.opCode, i); } i.held++; } return opCounts; }
static EditLogValidation scanEditLog(EditLogInputStream in) { long lastPos = 0; long lastTxId = HdfsConstants.INVALID_TXID; long numValid = 0; FSEditLogOp op = null; while (true) { lastPos = in.getPosition(); try { if ((op = in.readOp()) == null) { // TODO break; } } catch (Throwable t) { FSImage.LOG.warn("Caught exception after reading " + numValid + " ops from " + in + " while determining its valid length." + "Position was " + lastPos, t); in.resync(); FSImage.LOG.warn("After resync, position is " + in.getPosition()); continue; } if (lastTxId == HdfsConstants.INVALID_TXID || op.getTransactionId() > lastTxId) { lastTxId = op.getTransactionId(); } numValid++; } return new EditLogValidation(lastPos, lastTxId, false); }
expected++) { FSEditLogOp op = stream.readOp(); while (op == null) { assertTrue("Expected to find txid " + expected + ", " + iter.hasNext()); stream = iter.next(); op = stream.readOp(); assertNull(stream.readOp()); assertFalse("Expected no more txns after " + lastTxnId + " but more streams are available", iter.hasNext());
private static long readAllEdits(Collection<EditLogInputStream> streams, long startTxId) throws IOException { FSEditLogOp op; long nextTxId = startTxId; long numTx = 0; for (EditLogInputStream s : streams) { while (true) { op = s.readOp(); if (op == null) break; if (op.getTransactionId() != nextTxId) { throw new IOException("out of order transaction ID! expected " + nextTxId + " but got " + op.getTransactionId() + " when " + "reading " + s.getName()); } numTx++; nextTxId = op.getTransactionId() + 1; } } return numTx; }
/** * Make sure that we starting reading the correct op when we request a stream * with a txid in the middle of an edit log file. */ @Test public void testReadFromMiddleOfEditLog() throws CorruptionException, IOException { File f = new File(TestEditLog.TEST_DIR + "/readfrommiddleofeditlog"); NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10); StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next(); FileJournalManager jm = new FileJournalManager(conf, sd, storage); EditLogInputStream elis = getJournalInputStream(jm, 5, true); try { FSEditLogOp op = elis.readOp(); assertEquals("read unexpected op", op.getTransactionId(), 5); } finally { IOUtils.cleanup(LOG, elis); } }
static EditLogValidation scanEditLog(EditLogInputStream in) { long lastPos = 0; long lastTxId = HdfsConstants.INVALID_TXID; long numValid = 0; FSEditLogOp op = null; while (true) { lastPos = in.getPosition(); try { if ((op = in.readOp()) == null) { // TODO break; } } catch (Throwable t) { FSImage.LOG.warn("Caught exception after reading " + numValid + " ops from " + in + " while determining its valid length." + "Position was " + lastPos, t); in.resync(); FSImage.LOG.warn("After resync, position is " + in.getPosition()); continue; } if (lastTxId == HdfsConstants.INVALID_TXID || op.getTransactionId() > lastTxId) { lastTxId = op.getTransactionId(); } numValid++; } return new EditLogValidation(lastPos, lastTxId, false); }
/** * Make sure that in-progress streams aren't counted if we don't ask for * them. */ @Test public void testExcludeInProgressStreams() throws CorruptionException, IOException { File f = new File(TestEditLog.TEST_DIR + "/excludeinprogressstreams"); // Don't close the edit log once the files have been set up. NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10, false); StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next(); FileJournalManager jm = new FileJournalManager(conf, sd, storage); // If we exclude the in-progess stream, we should only have 100 tx. assertEquals(100, getNumberOfTransactions(jm, 1, false, false)); EditLogInputStream elis = getJournalInputStream(jm, 90, false); try { FSEditLogOp lastReadOp = null; while ((lastReadOp = elis.readOp()) != null) { assertTrue(lastReadOp.getTransactionId() <= 100); } } finally { IOUtils.cleanup(LOG, elis); } }