public static RemoteEditLogManifest convert( RemoteEditLogManifestProto manifest) { List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>(manifest .getLogsList().size()); for (RemoteEditLogProto l : manifest.getLogsList()) { logs.add(convert(l)); } return new RemoteEditLogManifest(logs, manifest.getCommittedTxnId()); }
if (manifest.getLogs().isEmpty()) { throw new IOException("Found no edit logs to download on NN since txid " + sig.mostRecentCheckpointTxId); if (manifest.getLogs().get(0).getStartTxId() != expectedTxId) { throw new IOException("Bad edit log manifest (expected txid = " + expectedTxId + ": " + manifest);
public RemoteEditLogManifest(List<RemoteEditLog> logs, long committedTxnId) { this.logs = logs; this.committedTxnId = committedTxnId; checkState(); }
public static RemoteEditLogManifestProto convert( RemoteEditLogManifest manifest) { RemoteEditLogManifestProto.Builder builder = RemoteEditLogManifestProto .newBuilder() .setCommittedTxnId(manifest.getCommittedTxnId()); for (RemoteEditLog log : manifest.getLogs()) { builder.addLogs(convert(log)); } return builder.build(); }
@Test public void testConvertRemoteEditLogManifest() { List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>(); logs.add(new RemoteEditLog(1, 10)); logs.add(new RemoteEditLog(11, 20)); RemoteEditLogManifest m = new RemoteEditLogManifest(logs); RemoteEditLogManifestProto mProto = PBHelper.convert(m); RemoteEditLogManifest m1 = PBHelper.convert(mProto); List<RemoteEditLog> logs1 = m1.getLogs(); assertEquals(logs.size(), logs1.size()); for (int i = 0; i < logs.size(); i++) { compare(logs.get(i), logs1.get(i)); } } public ExtendedBlock getExtendedBlock() {
log.initJournalsForWrite(); assertEquals("[[1,100], [101,200]]", log.getEditLogManifest(1).toString()); assertEquals("[[101,200]]", log.getEditLogManifest(101).toString()); log.initJournalsForWrite(); assertEquals("[[1,100], [101,200], [201,300], [301,400]]", log.getEditLogManifest(1).toString()); log.initJournalsForWrite(); assertEquals("[[301,400], [401,500]]", log.getEditLogManifest(1).toString()); log.initJournalsForWrite(); assertEquals("[[1,100], [101,200]]", log.getEditLogManifest(1).toString()); assertEquals("[[101,200]]", log.getEditLogManifest(101).toString()); log.initJournalsForWrite(); assertEquals("[[1,100], [101,200]]", log.getEditLogManifest(1).toString()); assertEquals("[[101,200]]", log.getEditLogManifest(101).toString());
AsyncLogger logger = e.getKey(); RemoteEditLogManifest manifest = e.getValue(); long committedTxnId = manifest.getCommittedTxnId(); for (RemoteEditLog remoteLog : manifest.getLogs()) { URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId());
private void syncWithJournalAtIndex(int index) { LOG.info("Syncing Journal " + jn.getBoundIpcAddress().getAddress() + ":" + jn.getBoundIpcAddress().getPort() + " with " + otherJNProxies.get(index) + ", journal id: " + jid); final InterQJournalProtocol jnProxy = otherJNProxies.get(index).jnProxy; if (jnProxy == null) { LOG.error("JournalNode Proxy not found."); return; } List<RemoteEditLog> thisJournalEditLogs; try { thisJournalEditLogs = journal.getEditLogManifest(0, false).getLogs(); } catch (IOException e) { LOG.error("Exception in getting local edit log manifest", e); return; } GetEditLogManifestResponseProto editLogManifest; try { editLogManifest = jnProxy.getEditLogManifestFromJournal(jid, nameServiceId, 0, false); } catch (IOException e) { LOG.error("Could not sync with Journal at " + otherJNProxies.get(journalNodeIndexForSync), e); return; } getMissingLogSegments(thisJournalEditLogs, editLogManifest, otherJNProxies.get(index)); }
RemoteEditLogManifest ret = new RemoteEditLogManifest(logs, curStartTxId - 1);
public RemoteEditLogManifest(List<RemoteEditLog> logs) { this.logs = logs; checkState(); }
response.getManifest()).getLogs(); if (otherJournalEditLogs == null || otherJournalEditLogs.isEmpty()) { LOG.warn("Journal at " + remoteJNproxy.jnAddr + " has no edit logs");
/** * @see QJournalProtocol#getEditLogManifest(String, String, long, boolean) */ public RemoteEditLogManifest getEditLogManifest(long sinceTxId, boolean inProgressOk) throws IOException { // No need to checkRequest() here - anyone may ask for the list // of segments. checkFormatted(); List<RemoteEditLog> logs = fjm.getRemoteEditLogs(sinceTxId, inProgressOk); if (inProgressOk) { RemoteEditLog log = null; for (Iterator<RemoteEditLog> iter = logs.iterator(); iter.hasNext();) { log = iter.next(); if (log.isInProgress()) { iter.remove(); break; } } if (log != null && log.isInProgress()) { logs.add(new RemoteEditLog(log.getStartTxId(), getHighestWrittenTxId(), true)); } } return new RemoteEditLogManifest(logs, getCommittedTxnId()); }
public RemoteEditLogManifest(List<RemoteEditLog> logs) { this.logs = logs; checkState(); }
static void rollForwardByApplyingLogs( RemoteEditLogManifest manifest, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); List<EditLogInputStream> editsStreams = Lists.newArrayList(); for (RemoteEditLog log : manifest.getLogs()) { if (log.getEndTxId() > dstImage.getLastAppliedTxId()) { File f = dstStorage.findFinalizedEditsFile( log.getStartTxId(), log.getEndTxId()); editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), log.getEndTxId(), true)); } } LOG.info("Checkpointer about to load edits from " + editsStreams.size() + " stream(s)."); dstImage.loadEdits(editsStreams, dstNamesystem); } }
public static RemoteEditLogManifest convert( RemoteEditLogManifestProto manifest) { List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>(manifest .getLogsList().size()); for (RemoteEditLogProto l : manifest.getLogsList()) { logs.add(convert(l)); } return new RemoteEditLogManifest(logs); }
@Override public Boolean run() throws Exception { dstImage.getStorage().cTime = sig.cTime; // get fsimage if (sig.mostRecentCheckpointTxId == dstImage.getStorage().getMostRecentCheckpointTxId()) { LOG.info("Image has not changed. Will not download image."); } else { LOG.info("Image has changed. Downloading updated image from NN."); MD5Hash downloadedHash = TransferFsImage.downloadImageToStorage( nnHostPort, sig.mostRecentCheckpointTxId, dstImage.getStorage(), true, false); dstImage.saveDigestAndRenameCheckpointImage(NameNodeFile.IMAGE, sig.mostRecentCheckpointTxId, downloadedHash); } // get edits file for (RemoteEditLog log : manifest.getLogs()) { TransferFsImage.downloadEditsToStorage( nnHostPort, log, dstImage.getStorage()); } // true if we haven't loaded all the transactions represented by the // downloaded fsimage. return dstImage.getLastAppliedTxId() < sig.mostRecentCheckpointTxId; } });
public static RemoteEditLogManifest convert( RemoteEditLogManifestProto manifest) { List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>(manifest .getLogsList().size()); for (RemoteEditLogProto l : manifest.getLogsList()) { logs.add(convert(l)); } return new RemoteEditLogManifest(logs); }
RemoteEditLogManifest ret = new RemoteEditLogManifest(logs);
if (manifest.getLogs().isEmpty()) { throw new IOException("Found no edit logs to download on NN since txid " + sig.mostRecentCheckpointTxId); if (manifest.getLogs().get(0).getStartTxId() != expectedTxId) { throw new IOException("Bad edit log manifest (expected txid = " + expectedTxId + ": " + manifest);