@Override public long length() throws IOException { return streams[curIdx].length(); }
/** * Creates a Step used for updating startup progress, populated with * information from the given edits. The step always includes the log's name. * If the log has a known length, then the length is included in the step too. * * @param edits EditLogInputStream to use for populating step * @return Step populated with information from edits * @throws IOException thrown if there is an I/O error */ private static Step createStartupProgressStep(EditLogInputStream edits) throws IOException { long length = edits.length(); String name = edits.getCurrentStreamName(); return length != -1 ? new Step(name, length) : new Step(name); } }
@Override public long length() throws IOException { return streams[curIdx].length(); }
/** * Load an edit log, and apply the changes to the in-memory structure * This is where we apply edits that we've been writing to disk all * along. */ long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId, long maxTxnsToRead, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); Step step = createStartupProgressStep(edits); prog.beginStep(Phase.LOADING_EDITS, step); fsNamesys.writeLock(); try { long startTime = monotonicNow(); FSImage.LOG.info("Start loading edits file " + edits.getName() + " maxTxnsToRead = " + maxTxnsToRead); long numEdits = loadEditRecords(edits, false, expectedStartingTxId, maxTxnsToRead, startOpt, recovery); FSImage.LOG.info("Edits file " + edits.getName() + " of size " + edits.length() + " edits # " + numEdits + " loaded in " + (monotonicNow()-startTime)/1000 + " seconds"); return numEdits; } finally { edits.close(); fsNamesys.writeUnlock("loadFSEdits"); prog.endStep(Phase.LOADING_EDITS, step); } }
@Override public long length() throws IOException { return streams[curIdx].length(); }
} catch (MetaRecoveryContext.RequestStopException e) { MetaRecoveryContext.LOG.warn("Stopped reading edit log at " + in.getPosition() + "/" + in.length()); break;
/** * Load an edit log, and apply the changes to the in-memory structure * This is where we apply edits that we've been writing to disk all * along. */ int loadFSEdits(EditLogInputStream edits, long expectedStartingTxId) throws IOException { long startTime = now(); currentTxId = expectedStartingTxId; int numEdits = loadFSEdits(edits, true); FSImage.LOG.info("Edits file " + edits.getName() + " of size " + edits.length() + " edits # " + numEdits + " loaded in " + (now()-startTime)/1000 + " seconds."); return numEdits; }
/** * Creates a Step used for updating startup progress, populated with * information from the given edits. The step always includes the log's name. * If the log has a known length, then the length is included in the step too. * * @param edits EditLogInputStream to use for populating step * @return Step populated with information from edits * @throws IOException thrown if there is an I/O error */ private static Step createStartupProgressStep(EditLogInputStream edits) throws IOException { long length = edits.length(); String name = edits.getCurrentStreamName(); return length != -1 ? new Step(name, length) : new Step(name); } }
/** * Creates a Step used for updating startup progress, populated with * information from the given edits. The step always includes the log's name. * If the log has a known length, then the length is included in the step too. * * @param edits EditLogInputStream to use for populating step * @return Step populated with information from edits * @throws IOException thrown if there is an I/O error */ private static Step createStartupProgressStep(EditLogInputStream edits) throws IOException { long length = edits.length(); String name = edits.getCurrentStreamName(); return length != -1 ? new Step(name, length) : new Step(name); } }
@Test public void testReadURL() throws Exception { HttpURLConnection conn = mock(HttpURLConnection.class); doReturn(new ByteArrayInputStream(FAKE_LOG_DATA)).when(conn).getInputStream(); doReturn(HttpURLConnection.HTTP_OK).when(conn).getResponseCode(); doReturn(Integer.toString(FAKE_LOG_DATA.length)).when(conn).getHeaderField("Content-Length"); URLConnectionFactory factory = mock(URLConnectionFactory.class); doReturn(conn).when(factory).openConnection(Mockito.<URL> any(), anyBoolean()); URL url = new URL("http://localhost/fakeLog"); EditLogInputStream elis = EditLogFileInputStream.fromUrl(factory, url, HdfsConstants.INVALID_TXID, HdfsConstants.INVALID_TXID, false); // Read the edit log and verify that we got all of the data. EnumMap<FSEditLogOpCodes, Holder<Integer>> counts = FSImageTestUtil .countEditLogOpTypes(elis); assertThat(counts.get(FSEditLogOpCodes.OP_ADD).held, is(1)); assertThat(counts.get(FSEditLogOpCodes.OP_SET_GENSTAMP_V1).held, is(1)); assertThat(counts.get(FSEditLogOpCodes.OP_CLOSE).held, is(1)); // Check that length header was picked up. assertEquals(FAKE_LOG_DATA.length, elis.length()); elis.close(); } }
/** * Load an edit log, and apply the changes to the in-memory structure * This is where we apply edits that we've been writing to disk all * along. */ long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); Step step = createStartupProgressStep(edits); prog.beginStep(Phase.LOADING_EDITS, step); fsNamesys.writeLock(); try { long startTime = monotonicNow(); FSImage.LOG.info("Start loading edits file " + edits.getName()); long numEdits = loadEditRecords(edits, false, expectedStartingTxId, startOpt, recovery); FSImage.LOG.info("Edits file " + edits.getName() + " of size " + edits.length() + " edits # " + numEdits + " loaded in " + (monotonicNow()-startTime)/1000 + " seconds"); return numEdits; } finally { edits.close(); fsNamesys.writeUnlock("loadFSEdits"); prog.endStep(Phase.LOADING_EDITS, step); } }
} catch (MetaRecoveryContext.RequestStopException e) { MetaRecoveryContext.LOG.warn("Stopped reading edit log at " + in.getPosition() + "/" + in.length()); break;
+ " of size " + edits.length() + " edits # " + numEdits + " loaded in " + (FSNamesystem.now()-startTime)/1000 + " seconds.");
/** * Load an edit log, and apply the changes to the in-memory structure * This is where we apply edits that we've been writing to disk all * along. */ long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); Step step = createStartupProgressStep(edits); prog.beginStep(Phase.LOADING_EDITS, step); fsNamesys.writeLock(); try { long startTime = monotonicNow(); FSImage.LOG.info("Start loading edits file " + edits.getName()); long numEdits = loadEditRecords(edits, false, expectedStartingTxId, startOpt, recovery); FSImage.LOG.info("Edits file " + edits.getName() + " of size " + edits.length() + " edits # " + numEdits + " loaded in " + (monotonicNow()-startTime)/1000 + " seconds"); return numEdits; } finally { edits.close(); fsNamesys.writeUnlock("loadFSEdits"); prog.endStep(Phase.LOADING_EDITS, step); } }
} catch (MetaRecoveryContext.RequestStopException e) { MetaRecoveryContext.LOG.warn("Stopped reading edit log at " + in.getPosition() + "/" + in.length()); break;