static OfflineEditsLoader createLoader(OfflineEditsVisitor visitor, String inputFileName, boolean xmlInput, OfflineEditsViewer.Flags flags) throws IOException { if (xmlInput) { return new OfflineEditsXmlLoader(visitor, new File(inputFileName), flags); } else { File file = null; EditLogInputStream elis = null; OfflineEditsLoader loader = null; try { file = new File(inputFileName); elis = new EditLogFileInputStream(file, HdfsServerConstants.INVALID_TXID, HdfsServerConstants.INVALID_TXID, false); loader = new OfflineEditsBinaryLoader(visitor, elis, flags); } finally { if ((loader == null) && (elis != null)) { elis.close(); } } return loader; } } }
/** * Load an edit log, and apply the changes to the in-memory structure * This is where we apply edits that we've been writing to disk all * along. */ long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId, long maxTxnsToRead, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); Step step = createStartupProgressStep(edits); prog.beginStep(Phase.LOADING_EDITS, step); fsNamesys.writeLock(); try { long startTime = monotonicNow(); FSImage.LOG.info("Start loading edits file " + edits.getName() + " maxTxnsToRead = " + maxTxnsToRead); long numEdits = loadEditRecords(edits, false, expectedStartingTxId, maxTxnsToRead, startOpt, recovery); FSImage.LOG.info("Edits file " + edits.getName() + " of size " + edits.length() + " edits # " + numEdits + " loaded in " + (monotonicNow()-startTime)/1000 + " seconds"); return numEdits; } finally { edits.close(); fsNamesys.writeUnlock("loadFSEdits"); prog.endStep(Phase.LOADING_EDITS, step); } }
fsNamesys.dir.resetLastInodeId(lastInodeId); if(closeOnExit) { in.close();
int loadFSEdits(EditLogInputStream edits, boolean closeOnExit) throws IOException { int numEdits = 0; int logVersion = edits.getVersion(); try { numEdits = loadEditRecords(logVersion, edits, false); } finally { if(closeOnExit) { edits.close(); } } if (logVersion != FSConstants.LAYOUT_VERSION) // other version numEdits++; // save this image asap return numEdits; }
static OfflineEditsLoader createLoader(OfflineEditsVisitor visitor, String inputFileName, boolean xmlInput, OfflineEditsViewer.Flags flags) throws IOException { if (xmlInput) { return new OfflineEditsXmlLoader(visitor, new File(inputFileName), flags); } else { File file = null; EditLogInputStream elis = null; OfflineEditsLoader loader = null; try { file = new File(inputFileName); elis = new EditLogFileInputStream(file, HdfsConstants.INVALID_TXID, HdfsConstants.INVALID_TXID, false); loader = new OfflineEditsBinaryLoader(visitor, elis, flags); } finally { if ((loader == null) && (elis != null)) { elis.close(); } } return loader; } } }
static OfflineEditsLoader createLoader(OfflineEditsVisitor visitor, String inputFileName, boolean xmlInput, OfflineEditsViewer.Flags flags) throws IOException { if (xmlInput) { return new OfflineEditsXmlLoader(visitor, new File(inputFileName), flags); } else { File file = null; EditLogInputStream elis = null; OfflineEditsLoader loader = null; try { file = new File(inputFileName); elis = new EditLogFileInputStream(file, HdfsConstants.INVALID_TXID, HdfsConstants.INVALID_TXID, false); loader = new OfflineEditsBinaryLoader(visitor, elis, flags); } finally { if ((loader == null) && (elis != null)) { elis.close(); } } return loader; } } }
@Test public void testReadURL() throws Exception { HttpURLConnection conn = mock(HttpURLConnection.class); doReturn(new ByteArrayInputStream(FAKE_LOG_DATA)).when(conn).getInputStream(); doReturn(HttpURLConnection.HTTP_OK).when(conn).getResponseCode(); doReturn(Integer.toString(FAKE_LOG_DATA.length)).when(conn).getHeaderField("Content-Length"); URLConnectionFactory factory = mock(URLConnectionFactory.class); doReturn(conn).when(factory).openConnection(Mockito.<URL> any(), anyBoolean()); URL url = new URL("http://localhost/fakeLog"); EditLogInputStream elis = EditLogFileInputStream.fromUrl(factory, url, HdfsConstants.INVALID_TXID, HdfsConstants.INVALID_TXID, false); // Read the edit log and verify that we got all of the data. EnumMap<FSEditLogOpCodes, Holder<Integer>> counts = FSImageTestUtil .countEditLogOpTypes(elis); assertThat(counts.get(FSEditLogOpCodes.OP_ADD).held, is(1)); assertThat(counts.get(FSEditLogOpCodes.OP_SET_GENSTAMP_V1).held, is(1)); assertThat(counts.get(FSEditLogOpCodes.OP_CLOSE).held, is(1)); // Check that length header was picked up. assertEquals(FAKE_LOG_DATA.length, elis.length()); elis.close(); } }
/** * Load an edit log, and apply the changes to the in-memory structure * This is where we apply edits that we've been writing to disk all * along. */ long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); Step step = createStartupProgressStep(edits); prog.beginStep(Phase.LOADING_EDITS, step); fsNamesys.writeLock(); try { long startTime = monotonicNow(); FSImage.LOG.info("Start loading edits file " + edits.getName()); long numEdits = loadEditRecords(edits, false, expectedStartingTxId, startOpt, recovery); FSImage.LOG.info("Edits file " + edits.getName() + " of size " + edits.length() + " edits # " + numEdits + " loaded in " + (monotonicNow()-startTime)/1000 + " seconds"); return numEdits; } finally { edits.close(); fsNamesys.writeUnlock("loadFSEdits"); prog.endStep(Phase.LOADING_EDITS, step); } }
private static EditLogInputStream getJournalInputStream(JournalManager jm, long txId, boolean inProgressOk) throws IOException { final PriorityQueue<EditLogInputStream> allStreams = new PriorityQueue<EditLogInputStream>(64, JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR); jm.selectInputStreams(allStreams, txId, inProgressOk); EditLogInputStream elis = null, ret; try { while ((elis = allStreams.poll()) != null) { if (elis.getFirstTxId() > txId) { break; } if (elis.getLastTxId() < txId) { elis.close(); continue; } elis.skipUntil(txId); ret = elis; elis = null; return ret; } } finally { IOUtils.cleanup(LOG, allStreams.toArray(new EditLogInputStream[0])); IOUtils.cleanup(LOG, elis); } return null; }
fsNamesys.dir.resetLastInodeId(lastInodeId); if(closeOnExit) { in.close();
/** * Load an edit log, and apply the changes to the in-memory structure * This is where we apply edits that we've been writing to disk all * along. */ long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); Step step = createStartupProgressStep(edits); prog.beginStep(Phase.LOADING_EDITS, step); fsNamesys.writeLock(); try { long startTime = monotonicNow(); FSImage.LOG.info("Start loading edits file " + edits.getName()); long numEdits = loadEditRecords(edits, false, expectedStartingTxId, startOpt, recovery); FSImage.LOG.info("Edits file " + edits.getName() + " of size " + edits.length() + " edits # " + numEdits + " loaded in " + (monotonicNow()-startTime)/1000 + " seconds"); return numEdits; } finally { edits.close(); fsNamesys.writeUnlock("loadFSEdits"); prog.endStep(Phase.LOADING_EDITS, step); } }
fsNamesys.dir.resetLastInodeId(lastInodeId); if(closeOnExit) { in.close();