if (manifest.getLogs().isEmpty()) { throw new IOException("Found no edit logs to download on NN since txid " + sig.mostRecentCheckpointTxId); if (manifest.getLogs().get(0).getStartTxId() != expectedTxId) { throw new IOException("Bad edit log manifest (expected txid = " + expectedTxId + ": " + manifest);
private void syncWithJournalAtIndex(int index) { LOG.info("Syncing Journal " + jn.getBoundIpcAddress().getAddress() + ":" + jn.getBoundIpcAddress().getPort() + " with " + otherJNProxies.get(index) + ", journal id: " + jid); final InterQJournalProtocol jnProxy = otherJNProxies.get(index).jnProxy; if (jnProxy == null) { LOG.error("JournalNode Proxy not found."); return; } List<RemoteEditLog> thisJournalEditLogs; try { thisJournalEditLogs = journal.getEditLogManifest(0, false).getLogs(); } catch (IOException e) { LOG.error("Exception in getting local edit log manifest", e); return; } GetEditLogManifestResponseProto editLogManifest; try { editLogManifest = jnProxy.getEditLogManifestFromJournal(jid, nameServiceId, 0, false); } catch (IOException e) { LOG.error("Could not sync with Journal at " + otherJNProxies.get(journalNodeIndexForSync), e); return; } getMissingLogSegments(thisJournalEditLogs, editLogManifest, otherJNProxies.get(index)); }
response.getManifest()).getLogs(); if (otherJournalEditLogs == null || otherJournalEditLogs.isEmpty()) { LOG.warn("Journal at " + remoteJNproxy.jnAddr + " has no edit logs");
static void rollForwardByApplyingLogs( RemoteEditLogManifest manifest, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); List<EditLogInputStream> editsStreams = Lists.newArrayList(); for (RemoteEditLog log : manifest.getLogs()) { if (log.getEndTxId() > dstImage.getLastAppliedTxId()) { File f = dstStorage.findFinalizedEditsFile( log.getStartTxId(), log.getEndTxId()); editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), log.getEndTxId(), true)); } } LOG.info("Checkpointer about to load edits from " + editsStreams.size() + " stream(s)."); dstImage.loadEdits(editsStreams, dstNamesystem); } }
long committedTxnId = manifest.getCommittedTxnId(); for (RemoteEditLog remoteLog : manifest.getLogs()) { URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId());
public static RemoteEditLogManifestProto convert( RemoteEditLogManifest manifest) { RemoteEditLogManifestProto.Builder builder = RemoteEditLogManifestProto .newBuilder() .setCommittedTxnId(manifest.getCommittedTxnId()); for (RemoteEditLog log : manifest.getLogs()) { builder.addLogs(convert(log)); } return builder.build(); }
@Override public Boolean run() throws Exception { dstImage.getStorage().cTime = sig.cTime; // get fsimage if (sig.mostRecentCheckpointTxId == dstImage.getStorage().getMostRecentCheckpointTxId()) { LOG.info("Image has not changed. Will not download image."); } else { LOG.info("Image has changed. Downloading updated image from NN."); MD5Hash downloadedHash = TransferFsImage.downloadImageToStorage( nnHostPort, sig.mostRecentCheckpointTxId, dstImage.getStorage(), true, false); dstImage.saveDigestAndRenameCheckpointImage(NameNodeFile.IMAGE, sig.mostRecentCheckpointTxId, downloadedHash); } // get edits file for (RemoteEditLog log : manifest.getLogs()) { TransferFsImage.downloadEditsToStorage( nnHostPort, log, dstImage.getStorage()); } // true if we haven't loaded all the transactions represented by the // downloaded fsimage. return dstImage.getLastAppliedTxId() < sig.mostRecentCheckpointTxId; } });
if (manifest.getLogs().isEmpty()) { throw new IOException("Found no edit logs to download on NN since txid " + sig.mostRecentCheckpointTxId); if (manifest.getLogs().get(0).getStartTxId() != expectedTxId) { throw new IOException("Bad edit log manifest (expected txid = " + expectedTxId + ": " + manifest);
if (manifest.getLogs().isEmpty()) { throw new IOException("Found no edit logs to download on NN since txid " + sig.mostRecentCheckpointTxId); if (manifest.getLogs().get(0).getStartTxId() != expectedTxId) { throw new IOException("Bad edit log manifest (expected txid = " + expectedTxId + ": " + manifest);
@Override public void selectInputStreams(Collection<EditLogInputStream> streams, long fromTxnId, boolean inProgressOk) throws IOException { QuorumCall<AsyncLogger, RemoteEditLogManifest> q = loggers.getEditLogManifest(fromTxnId, inProgressOk); Map<AsyncLogger, RemoteEditLogManifest> resps = loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs, "selectInputStreams"); LOG.debug("selectInputStream manifests:\n" + Joiner.on("\n").withKeyValueSeparator(": ").join(resps)); final PriorityQueue<EditLogInputStream> allStreams = new PriorityQueue<EditLogInputStream>(64, JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR); for (Map.Entry<AsyncLogger, RemoteEditLogManifest> e : resps.entrySet()) { AsyncLogger logger = e.getKey(); RemoteEditLogManifest manifest = e.getValue(); for (RemoteEditLog remoteLog : manifest.getLogs()) { URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId()); EditLogInputStream elis = EditLogFileInputStream.fromUrl( connectionFactory, url, remoteLog.getStartTxId(), remoteLog.getEndTxId(), remoteLog.isInProgress()); allStreams.add(elis); } } JournalSet.chainAndMakeRedundantStreams(streams, allStreams, fromTxnId); }
public static RemoteEditLogManifestProto convert( RemoteEditLogManifest manifest) { RemoteEditLogManifestProto.Builder builder = RemoteEditLogManifestProto .newBuilder(); for (RemoteEditLog log : manifest.getLogs()) { builder.addLogs(convert(log)); } return builder.build(); }
public static RemoteEditLogManifestProto convert( RemoteEditLogManifest manifest) { RemoteEditLogManifestProto.Builder builder = RemoteEditLogManifestProto .newBuilder(); for (RemoteEditLog log : manifest.getLogs()) { builder.addLogs(convert(log)); } return builder.build(); }
static void rollForwardByApplyingLogs( RemoteEditLogManifest manifest, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); List<EditLogInputStream> editsStreams = Lists.newArrayList(); for (RemoteEditLog log : manifest.getLogs()) { if (log.getEndTxId() > dstImage.getLastAppliedTxId()) { File f = dstStorage.findFinalizedEditsFile( log.getStartTxId(), log.getEndTxId()); editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), log.getEndTxId(), true)); } } LOG.info("Checkpointer about to load edits from " + editsStreams.size() + " stream(s)."); dstImage.loadEdits(editsStreams, dstNamesystem); } }
@Override public void selectInputStreams(Collection<EditLogInputStream> streams, long fromTxnId, boolean inProgressOk) throws IOException { QuorumCall<AsyncLogger, RemoteEditLogManifest> q = loggers.getEditLogManifest(fromTxnId, inProgressOk); Map<AsyncLogger, RemoteEditLogManifest> resps = loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs, "selectInputStreams"); LOG.debug("selectInputStream manifests:\n" + Joiner.on("\n").withKeyValueSeparator(": ").join(resps)); final PriorityQueue<EditLogInputStream> allStreams = new PriorityQueue<EditLogInputStream>(64, JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR); for (Map.Entry<AsyncLogger, RemoteEditLogManifest> e : resps.entrySet()) { AsyncLogger logger = e.getKey(); RemoteEditLogManifest manifest = e.getValue(); for (RemoteEditLog remoteLog : manifest.getLogs()) { URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId()); EditLogInputStream elis = EditLogFileInputStream.fromUrl( connectionFactory, url, remoteLog.getStartTxId(), remoteLog.getEndTxId(), remoteLog.isInProgress()); allStreams.add(elis); } } JournalSet.chainAndMakeRedundantStreams(streams, allStreams, fromTxnId); }
@Test public void testConvertRemoteEditLogManifest() { List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>(); logs.add(new RemoteEditLog(1, 10)); logs.add(new RemoteEditLog(11, 20)); RemoteEditLogManifest m = new RemoteEditLogManifest(logs); RemoteEditLogManifestProto mProto = PBHelper.convert(m); RemoteEditLogManifest m1 = PBHelper.convert(mProto); List<RemoteEditLog> logs1 = m1.getLogs(); assertEquals(logs.size(), logs1.size()); for (int i = 0; i < logs.size(); i++) { compare(logs.get(i), logs1.get(i)); } } public ExtendedBlock getExtendedBlock() {
@Override public Boolean run() throws Exception { dstImage.getStorage().cTime = sig.cTime; // get fsimage if (sig.mostRecentCheckpointTxId == dstImage.getStorage().getMostRecentCheckpointTxId()) { LOG.info("Image has not changed. Will not download image."); } else { LOG.info("Image has changed. Downloading updated image from NN."); MD5Hash downloadedHash = TransferFsImage.downloadImageToStorage( nnHostPort, sig.mostRecentCheckpointTxId, dstImage.getStorage(), true); dstImage.saveDigestAndRenameCheckpointImage(NameNodeFile.IMAGE, sig.mostRecentCheckpointTxId, downloadedHash); } // get edits file for (RemoteEditLog log : manifest.getLogs()) { TransferFsImage.downloadEditsToStorage( nnHostPort, log, dstImage.getStorage()); } // true if we haven't loaded all the transactions represented by the // downloaded fsimage. return dstImage.getLastAppliedTxId() < sig.mostRecentCheckpointTxId; } });
static void rollForwardByApplyingLogs( RemoteEditLogManifest manifest, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); List<EditLogInputStream> editsStreams = Lists.newArrayList(); for (RemoteEditLog log : manifest.getLogs()) { if (log.getEndTxId() > dstImage.getLastAppliedTxId()) { File f = dstStorage.findFinalizedEditsFile( log.getStartTxId(), log.getEndTxId()); editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), log.getEndTxId(), true)); } } LOG.info("Checkpointer about to load edits from " + editsStreams.size() + " stream(s)."); dstImage.loadEdits(editsStreams, dstNamesystem); } }
@Override public Boolean run() throws Exception { dstImage.getStorage().cTime = sig.cTime; // get fsimage if (sig.mostRecentCheckpointTxId == dstImage.getStorage().getMostRecentCheckpointTxId()) { LOG.info("Image has not changed. Will not download image."); } else { LOG.info("Image has changed. Downloading updated image from NN."); MD5Hash downloadedHash = TransferFsImage.downloadImageToStorage( nnHostPort, sig.mostRecentCheckpointTxId, dstImage.getStorage(), true); dstImage.saveDigestAndRenameCheckpointImage(NameNodeFile.IMAGE, sig.mostRecentCheckpointTxId, downloadedHash); } // get edits file for (RemoteEditLog log : manifest.getLogs()) { TransferFsImage.downloadEditsToStorage( nnHostPort, log, dstImage.getStorage()); } // true if we haven't loaded all the transactions represented by the // downloaded fsimage. return dstImage.getLastAppliedTxId() < sig.mostRecentCheckpointTxId; } });
RemoteEditLog log = manifest.getLogs().get(0);