@Override public FSEditLogOp decodeOp() throws IOException { limiter.setLimit(maxOpSize); in.mark(maxOpSize); byte opCodeByte; try { opCodeByte = in.readByte(); } catch (EOFException eof) { // EOF at an opcode boundary is expected. return null; } FSEditLogOpCodes opCode = FSEditLogOpCodes.fromByte(opCodeByte); if (opCode == OP_INVALID) { verifyTerminator(); return null; } FSEditLogOp op = cache.get(opCode); if (op == null) { throw new IOException("Read invalid opcode " + opCode); } if (NameNodeLayoutVersion.supports( LayoutVersion.Feature.STORED_TXIDS, logVersion)) { op.setTransactionId(in.readLong()); } else { op.setTransactionId(HdfsServerConstants.INVALID_TXID); } op.readFields(in, logVersion); return op; }
opCode = null; try { op.decodeXml(stanza); stanza = null; } finally { nextTxId = op.getTransactionId(); if (nextTxId <= 0) { nextTxId = 1; op.setTransactionId(nextTxId); nextTxId++;
@Override void fromXml(Stanza st) throws InvalidXmlException { this.path = st.getValue("PATH"); List<Stanza> blocks = st.getChildren("BLOCK"); int size = blocks.size(); Preconditions.checkState(size == 1 || size == 2); this.penultimateBlock = size == 2 ? FSEditLogOp.blockFromXml(blocks.get(0)) : null; this.lastBlock = FSEditLogOp.blockFromXml(blocks.get(size - 1)); readRpcIdsFromXml(st); } }
synchronized boolean doEditTransaction(final FSEditLogOp op) { long start = beginTransaction(); op.setTransactionId(txid); try { editLogStream.write(op); } catch (IOException ex) { // All journals failed, it is handled in logSync. } finally { op.reset(); } endTransaction(start); return shouldForceSync(); }
Boolean.toString(overwrite)); for (Block b : blocks) { FSEditLogOp.blockToXml(contentHandler, b); FSEditLogOp.permissionStatusToXml(contentHandler, permissions); if (this.opCode == OP_ADD) { if (aclEntries != null) {
@Override public long scanOp() throws IOException { // Edit logs of this age don't have any length prefix, so we just have // to read the entire Op. FSEditLogOp op = decodeOp(); return op == null ? HdfsServerConstants.INVALID_TXID : op.getTransactionId(); } }
case OPEN: op = reader.readOp(skipBrokenEdits); if ((op != null) && (op.hasTransactionId())) { long txId = op.getTransactionId(); if ((txId >= lastTxId) && (lastTxId != HdfsServerConstants.INVALID_TXID)) {
if (fixTxIds) { if (nextTxId <= 0) { nextTxId = op.getTransactionId(); if (nextTxId <= 0) { nextTxId = 1; op.setTransactionId(nextTxId); nextTxId++;
public static void writeOp(EditLogOutputStream stm, long txid) throws IOException { FSEditLogOp op = NameNodeAdapter.createMkdirOp("tx " + txid); op.setTransactionId(txid); stm.write(op); }
@Override protected void toXml(ContentHandler contentHandler) throws SAXException { XMLUtils.addSaxString(contentHandler, "PATH", path); if (penultimateBlock != null) { FSEditLogOp.blockToXml(contentHandler, penultimateBlock); } FSEditLogOp.blockToXml(contentHandler, lastBlock); appendRpcIdsToXml(contentHandler, rpcClientId, rpcCallId); }
@Override protected void toXml(ContentHandler contentHandler) throws SAXException { FSEditLogOp.delegationTokenToXml(contentHandler, token); }
/** * Write an operation to the output stream * * @param op The operation to write * @throws IOException if an error occurs during writing. */ public void writeOp(FSEditLogOp op) throws IOException { int start = buf.getLength(); // write the op code first to make padding and terminator verification // work buf.writeByte(op.opCode.getOpCode()); buf.writeInt(0); // write 0 for the length first buf.writeLong(op.txid); op.writeFields(buf); int end = buf.getLength(); // write the length back: content of the op + 4 bytes checksum - op_code int length = end - start - 1; buf.writeInt(length, start + 1); checksum.reset(); checksum.update(buf.getData(), start, end-start); int sum = (int)checksum.getValue(); buf.writeInt(sum); } }
public void decodeXml(Stanza st) throws InvalidXmlException { this.txid = Long.parseLong(st.getValue("TXID")); fromXml(st); }
@Override protected void toXml(ContentHandler contentHandler) throws SAXException { FSEditLogOp.delegationKeyToXml(contentHandler, key); }
@Override public long scanOp() throws IOException { if (!NameNodeLayoutVersion.supports( LayoutVersion.Feature.STORED_TXIDS, logVersion)) { throw new IOException("Can't scan a pre-transactional edit log."); } FSEditLogOp op = decodeOp(); return op == null ? HdfsServerConstants.INVALID_TXID : op.getTransactionId(); } }
if (op.hasTransactionId()) { if (op.getTransactionId() > expectedTxId) { MetaRecoveryContext.editLogLoaderPrompt("There appears " + "to be a gap in the edit log. We expected txid " + expectedTxId + ", but got txid " + op.getTransactionId() + ".", recovery, "ignoring missing " + " transaction IDs"); } else if (op.getTransactionId() < expectedTxId) { MetaRecoveryContext.editLogLoaderPrompt("There appears " + "to be an out-of-order edit in the edit log. We " + "expected txid " + expectedTxId + ", but got txid " + op.getTransactionId() + ".", recovery, "skipping the out-of-order edit"); continue; if (op.hasTransactionId()) { lastAppliedTxId = op.getTransactionId(); expectedTxId = lastAppliedTxId + 1; } else { if (op.hasTransactionId()) { long now = monotonicNow(); if (now - lastLogTime > REPLAY_TRANSACTION_LOG_INTERVAL) {
if (fixTxIds) { if (nextTxId <= 0) { nextTxId = op.getTransactionId(); if (nextTxId <= 0) { nextTxId = 1; op.setTransactionId(nextTxId); nextTxId++;
op.setTransactionId(txid); op.reset();
@Override protected void toXml(ContentHandler contentHandler) throws SAXException { XMLUtils.addSaxString(contentHandler, "LENGTH", Integer.toString(length)); XMLUtils.addSaxString(contentHandler, "INODEID", Long.toString(inodeId)); XMLUtils.addSaxString(contentHandler, "PATH", path); XMLUtils.addSaxString(contentHandler, "REPLICATION", Short.valueOf(replication).toString()); XMLUtils.addSaxString(contentHandler, "MTIME", Long.toString(mtime)); XMLUtils.addSaxString(contentHandler, "ATIME", Long.toString(atime)); XMLUtils.addSaxString(contentHandler, "BLOCKSIZE", Long.toString(blockSize)); XMLUtils.addSaxString(contentHandler, "CLIENT_NAME", clientName); XMLUtils.addSaxString(contentHandler, "CLIENT_MACHINE", clientMachine); XMLUtils.addSaxString(contentHandler, "OVERWRITE", Boolean.toString(overwrite)); for (Block b : blocks) { FSEditLogOp.blockToXml(contentHandler, b); } FSEditLogOp.permissionStatusToXml(contentHandler, permissions); if (this.opCode == OP_ADD) { if (aclEntries != null) { appendAclEntriesToXml(contentHandler, aclEntries); } appendRpcIdsToXml(contentHandler, rpcClientId, rpcCallId); } }
op.setTransactionId(txid); assert this.getNumEditStreams() > 0 : "no editlog streams"; long start = FSNamesystem.now();