public void updateLastAppliedTxIdFromWritten() { this.lastAppliedTxId = editLog.getLastWrittenTxId(); }
/** * This method holds a lock of FSEditLog to get the correct value. * This method must not be used for metrics. */ public long getCorrectLastAppliedOrWrittenTxId() { return Math.max(lastAppliedTxId, editLog != null ? editLog.getLastWrittenTxId() : 0); }
/** * Blocks until all ongoing edits have been synced to disk. * This differs from logSync in that it waits for edits that have been * written by other threads, not just edits from the calling thread. * * NOTE: this should be done while holding the FSNamesystem lock, or * else more operations can start writing while this is in progress. */ void logSyncAll() { // Make sure we're synced up to the most recent transaction ID. long lastWrittenTxId = getLastWrittenTxId(); LOG.info("logSyncAll toSyncToTxId=" + lastWrittenTxId + " lastSyncedTxid=" + synctxid + " mostRecentTxid=" + txid); logSync(lastWrittenTxId); lastWrittenTxId = getLastWrittenTxId(); LOG.info("Done logSyncAll lastWrittenTxId=" + lastWrittenTxId + " lastSyncedTxid=" + synctxid + " mostRecentTxid=" + txid); }
/** * End checkpoint. * <p> * Validate the current storage info with the given signature. * * @param sig to validate the current storage info against * @throws IOException if the checkpoint fields are inconsistent */ void endCheckpoint(CheckpointSignature sig) throws IOException { LOG.info("End checkpoint at txid " + getEditLog().getLastWrittenTxId()); sig.validateStorageInfo(this); }
/** * Finalizes the current edit log and opens a new log segment. * * @param layoutVersion The layout version of the new edit log segment. * @return the transaction id of the BEGIN_LOG_SEGMENT transaction in the new * log. */ synchronized long rollEditLog(int layoutVersion) throws IOException { LOG.info("Rolling edit logs"); endCurrentLogSegment(true); long nextTxId = getLastWrittenTxId() + 1; startLogSegmentAndWriteHeaderTxn(nextTxId, layoutVersion); assert curSegmentTxId == nextTxId; return nextTxId; }
/** * Initialize the output stream for logging, opening the first * log segment. */ synchronized void openForWrite(int layoutVersion) throws IOException { Preconditions.checkState(state == State.BETWEEN_LOG_SEGMENTS, "Bad state: %s", state); long segmentTxId = getLastWrittenTxId() + 1; // Safety check: we should never start a segment if there are // newer txids readable. List<EditLogInputStream> streams = new ArrayList<EditLogInputStream>(); journalSet.selectInputStreams(streams, segmentTxId, true, false); if (!streams.isEmpty()) { String error = String.format("Cannot start writing at txid %s " + "when there is a stream available for read: %s", segmentTxId, streams.get(0)); IOUtils.cleanupWithLogger(LOG, streams.toArray(new EditLogInputStream[0])); throw new IllegalStateException(error); } startLogSegmentAndWriteHeaderTxn(segmentTxId, layoutVersion); assert state == State.IN_SEGMENT : "Bad state: " + state; }
/** Write the batch of edits to edit log. */ public synchronized void journal(long firstTxId, int numTxns, byte[] data) { final long expectedTxId = getLastWrittenTxId() + 1; Preconditions.checkState(firstTxId == expectedTxId, "received txid batch starting at %s but expected txid %s", firstTxId, expectedTxId); setNextTxId(firstTxId + numTxns - 1); logEdit(data.length, data); logSync(); }
/** * Remote namenode just has started a log segment, start log segment locally. */ public synchronized void startLogSegment(long txid, boolean abortCurrentLogSegment, int layoutVersion) throws IOException { LOG.info("Started a new log segment at txid " + txid); if (isSegmentOpen()) { if (getLastWrittenTxId() == txid - 1) { //In sync with the NN, so end and finalize the current segment` endCurrentLogSegment(false); } else { //Missed some transactions: probably lost contact with NN temporarily. final String mess = "Cannot start a new log segment at txid " + txid + " since only up to txid " + getLastWrittenTxId() + " have been written in the log segment starting at " + getCurSegmentTxId() + "."; if (abortCurrentLogSegment) { //Mark the current segment as aborted. LOG.warn(mess); abortCurrentLogSegment(); } else { throw new IOException(mess); } } } setNextTxId(txid); startLogSegment(txid, layoutVersion); }
", " + getLastWrittenTxId()); Preconditions.checkState(isSegmentOpen(), "Bad state: %s", state); final long lastTxId = getLastWrittenTxId(); final long lastSyncedTxId = getSyncTxId(); Preconditions.checkArgument(lastTxId == lastSyncedTxId,
@Override // ClientProtocol public long getCurrentEditLogTxid() throws IOException { checkNNStartup(); namesystem.checkOperation(OperationCategory.READ); // only active namesystem.checkSuperuserPrivilege(); // if it's not yet open for write, we may be in the process of transitioning // from standby to active and may not yet know what the latest committed // txid is return namesystem.getEditLog().isOpenForWrite() ? namesystem.getEditLog().getLastWrittenTxId() : -1; }
/** * Get the correct number of transactions since last edit log roll. * This method holds a lock of FSEditLog and must not be used for metrics. */ private long getCorrectTransactionsSinceLastLogRoll() { if (isInStandbyState() || !getEditLog().isSegmentOpen()) { return 0; } else { return getEditLog().getLastWrittenTxId() - getEditLog().getCurSegmentTxId() + 1; } }
errorSDs.clear(); saveFSImageInAllDirs(target, editLog.getLastWrittenTxId());
realImage.getEditLog().setNextTxId(ckptImage.getEditLog().getLastWrittenTxId()+1); realImage.initEditLog(StartupOption.IMPORT);
long remainingTxns = getEditLog().getLastWrittenTxId() - lastAppliedTxId; loader.loadFSEdits(stream, lastAppliedTxId + 1); lastAppliedTxId = loader.getLastAppliedTxId(); assert lastAppliedTxId == getEditLog().getLastWrittenTxId(); } finally { FSEditLog.closeAllStreams(editStreams);
/** * This method holds a lock of FSEditLog to get the correct value. * This method must not be used for metrics. */ public long getCorrectLastAppliedOrWrittenTxId() { return Math.max(lastAppliedTxId, editLog != null ? editLog.getLastWrittenTxId() : 0); }
int layoutVersion) // active name-node throws IOException { LOG.info("Start checkpoint at txid " + getEditLog().getLastWrittenTxId()); String msg = null;
/** * End checkpoint. * <p> * Validate the current storage info with the given signature. * * @param sig to validate the current storage info against * @throws IOException if the checkpoint fields are inconsistent */ void endCheckpoint(CheckpointSignature sig) throws IOException { LOG.info("End checkpoint at txid " + getEditLog().getLastWrittenTxId()); sig.validateStorageInfo(this); }
/** * Write the batch of edits to the local copy of the edit logs. */ private void logEditsLocally(long firstTxId, int numTxns, byte[] data) { long expectedTxId = editLog.getLastWrittenTxId() + 1; Preconditions.checkState(firstTxId == expectedTxId, "received txid batch starting at %s but expected txn %s", firstTxId, expectedTxId); editLog.setNextTxId(firstTxId + numTxns - 1); editLog.logEdit(data.length, data); editLog.logSync(); }
/** * End checkpoint. * <p> * Validate the current storage info with the given signature. * * @param sig to validate the current storage info against * @throws IOException if the checkpoint fields are inconsistent */ void endCheckpoint(CheckpointSignature sig) throws IOException { LOG.info("End checkpoint at txid " + getEditLog().getLastWrittenTxId()); sig.validateStorageInfo(this); }
public long getLastWrittenTxId() { if (getFSImage() != null && getFSImage().getEditLog() != null) { return getFSImage().getEditLog().getLastWrittenTxId(); } else { return -1; } }