checkConfiguration(conf); FSImage fsImage = new FSImage(conf, FSNamesystem.getNamespaceDirs(conf), FSNamesystem.getNamespaceEditsDirs(conf)); FSNamesystem namesystem = new FSNamesystem(conf, fsImage, false); StartupOption startOpt = NameNode.getStartupOption(conf); if (startOpt == StartupOption.RECOVER) { namesystem.setSafeMode(SafeModeAction.SAFEMODE_ENTER); namesystem.loadFSImage(startOpt); } catch (IOException ioe) { LOG.warn("Encountered exception loading fsimage", ioe); nnMetrics.setFsImageLoadTime((int) timeTakenToLoadFSImage); namesystem.getFSDirectory().createReservedStatuses(namesystem.getCTime()); return namesystem;
DFS_NAMENODE_AUDIT_LOG_ASYNC_DEFAULT)) { LOG.info("Enabling async auditlog"); enableAsyncAuditLog(); DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT); this.dtSecretManager = createDelegationTokenSecretManager(conf); this.dir = new FSDirectory(this, conf); this.snapshotManager = new SnapshotManager(conf, dir); this.auditLoggers = initAuditLoggers(conf); this.isDefaultAuditLogger = auditLoggers.size() == 1 && auditLoggers.get(0) instanceof DefaultAuditLogger; this.retryCache = ignoreRetryCache ? null : initRetryCache(conf); Class<? extends INodeAttributeProvider> klass = conf.getClass( DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_KEY, } catch(IOException e) { LOG.error(getClass().getSimpleName() + " initialization failed.", e); close(); throw e; } catch (RuntimeException re) { LOG.error(getClass().getSimpleName() + " initialization failed.", re); close(); throw re;
@Override // DatanodeProtocol public void commitBlockSynchronization(ExtendedBlock block, long newgenerationstamp, long newlength, boolean closeFile, boolean deleteblock, DatanodeID[] newtargets, String[] newtargetstorages) throws IOException { checkNNStartup(); namesystem.commitBlockSynchronization(block, newgenerationstamp, newlength, closeFile, deleteblock, newtargets, newtargetstorages); }
/** * Construct an FSImage * @param conf Configuration * @throws IOException if default directories are invalid. */ public FSImage(Configuration conf) throws IOException { this(conf, FSNamesystem.getNamespaceDirs(conf), FSNamesystem.getNamespaceEditsDirs(conf)); }
/** * Get the total number of COMPLETE blocks in the system. * For safe mode only complete blocks are counted. * This is invoked only during NN startup and checkpointing. */ public long getCompleteBlocksTotal() { // Calculate number of blocks under construction long numUCBlocks = 0; readLock(); try { numUCBlocks = leaseManager.getNumUnderConstructionBlocks(); return getBlocksTotal() - numUCBlocks; } finally { readUnlock("getCompleteBlocksTotal"); } }
boolean setSafeMode(SafeModeAction action) throws IOException { String operationName = action.toString().toLowerCase(); boolean error = false; if (action != SafeModeAction.SAFEMODE_GET) { checkSuperuserPrivilege(operationName); switch(action) { case SAFEMODE_LEAVE: // leave safe mode leaveSafeMode(false); break; case SAFEMODE_ENTER: // enter safe mode enterSafeMode(false); break; case SAFEMODE_FORCE_EXIT: leaveSafeMode(true); break; default: LOG.error("Unexpected safe mode action"); error = true; } } if (!error) { logAuditEvent(true, operationName, null); } return isInSafeMode(); }
filesTotal.set((int) fsNameSystem.getFilesAndDirectoriesTotal()); blocksTotal.set((int)fsNameSystem.getBlocksTotal()); diskSpaceTotalGB.set(roundBytesToGBytes(fsNameSystem.getDiskSpaceTotal())); capacityTotalGB.set(roundBytesToGBytes(fsNameSystem.getCapacityTotal())); capacityUsedGB.set(roundBytesToGBytes(fsNameSystem.getCapacityUsed())); capacityRemainingGB.set(roundBytesToGBytes(fsNameSystem. getCapacityRemaining())); totalLoad.set(fsNameSystem.getTotalLoad()); corruptBlocks.set((int)fsNameSystem.getCorruptReplicaBlocks()); excessBlocks.set((int)fsNameSystem.getExcessBlocks()); pendingDeletionBlocks.set((int)fsNameSystem.getPendingDeletionBlocks()); pendingReplicationBlocks.set((int)fsNameSystem. getPendingReplicationBlocks()); underReplicatedBlocks.set((int)fsNameSystem.getUnderReplicatedBlocks()); scheduledReplicationBlocks.set((int)fsNameSystem. getScheduledReplicationBlocks()); missingBlocks.set((int)fsNameSystem.getMissingBlocksCount()); blockCapacity.set(fsNameSystem.getBlockCapacity()); numLeases.set(fsNameSystem.leaseManager.countLease()); numUnderConstructionFiles.set(fsNameSystem.leaseManager.countPath()); upgradeTime.set(fsNameSystem.getUpgradeTime());
FSNamesystem fsNameSystem = FSNamesystem.getFSNamesystem(); filesTotal.set((int)fsNameSystem.getFilesTotal()); filesTotal.pushMetric(metricsRecord); blocksTotal.set((int)fsNameSystem.getBlocksTotal()); blocksTotal.pushMetric(metricsRecord); capacityTotalGB.set(roundBytesToGBytes(fsNameSystem.getCapacityTotal())); capacityTotalGB.pushMetric(metricsRecord); capacityUsedGB.set(roundBytesToGBytes(fsNameSystem.getCapacityUsed())); capacityUsedGB.pushMetric(metricsRecord); getCapacityRemaining())); capacityRemainingGB.pushMetric(metricsRecord); totalLoad.set(fsNameSystem.getTotalLoad()); totalLoad.pushMetric(metricsRecord); getPendingReplicationBlocks()); pendingReplicationBlocks.pushMetric(metricsRecord); underReplicatedBlocks.set((int)fsNameSystem.getUnderReplicatedBlocks()); underReplicatedBlocks.pushMetric(metricsRecord); getScheduledReplicationBlocks()); scheduledReplicationBlocks.pushMetric(metricsRecord);
@Override public void run () { try { while (fsRunning && shouldNNRmRun) { checkAvailableResources(); if(!nameNodeHasResourcesAvailable()) { String lowResourcesMsg = "NameNode low on available disk space. "; if (!isInSafeMode()) { LOG.warn(lowResourcesMsg + "Entering safe mode."); } else { LOG.warn(lowResourcesMsg + "Already in safe mode."); } enterSafeMode(true); } try { Thread.sleep(resourceRecheckInterval); } catch (InterruptedException ie) { // Deliberately ignore } } } catch (Exception e) { FSNamesystem.LOG.error("Exception in NameNodeResourceMonitor: ", e); } }
backupNode.namesystem.writeLock(); try { backupNode.namesystem.setImageLoaded(); if(backupNode.namesystem.getBlocksTotal() > 0) { long completeBlocksTotal = backupNode.namesystem.getCompleteBlocksTotal(); backupNode.namesystem.getBlockManager().setBlockTotal( completeBlocksTotal); backupNode.namesystem.writeUnlock("doCheckpoint");
@Test(timeout = 30000) public void testGetBlockLocationsRacingWithDelete() throws IOException { FSNamesystem fsn = spy(setupFileSystem()); final FSDirectory fsd = fsn.getFSDirectory(); FSEditLog editlog = fsn.getEditLog(); doAnswer(new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { INodesInPath iip = fsd.getINodesInPath(FILE_PATH, true); FSDirDeleteOp.delete(fsd, iip, new INode.BlocksMapUpdateInfo(), new ArrayList<INode>(), now()); invocation.callRealMethod(); return null; } }).when(fsn).writeLock(); fsn.getBlockLocations("dummy", RESERVED_PATH, 0, 1024); verify(editlog, never()).logTimes(anyString(), anyLong(), anyLong()); fsn.close(); }
@Test public void testReplQueuesActiveAfterStartupSafemode() throws IOException, InterruptedException{ Configuration conf = new Configuration(); FSEditLog fsEditLog = Mockito.mock(FSEditLog.class); FSImage fsImage = Mockito.mock(FSImage.class); Mockito.when(fsImage.getEditLog()).thenReturn(fsEditLog); FSNamesystem fsNamesystem = new FSNamesystem(conf, fsImage); FSNamesystem fsn = Mockito.spy(fsNamesystem); //Make shouldPopulaeReplQueues return true HAContext haContext = Mockito.mock(HAContext.class); HAState haState = Mockito.mock(HAState.class); Mockito.when(haContext.getState()).thenReturn(haState); Mockito.when(haState.shouldPopulateReplQueues()).thenReturn(true); Whitebox.setInternalState(fsn, "haContext", haContext); //Make NameNode.getNameNodeMetrics() not return null NameNode.initMetrics(conf, NamenodeRole.NAMENODE); fsn.enterSafeMode(false); assertTrue("FSNamesystem didn't enter safemode", fsn.isInSafeMode()); assertTrue("Replication queues were being populated during very first " + "safemode", !fsn.isPopulatingReplQueues()); fsn.leaveSafeMode(); assertTrue("FSNamesystem didn't leave safemode", !fsn.isInSafeMode()); assertTrue("Replication queues weren't being populated even after leaving " + "safemode", fsn.isPopulatingReplQueues()); fsn.enterSafeMode(false); assertTrue("FSNamesystem didn't enter safemode", fsn.isInSafeMode()); assertTrue("Replication queues weren't being populated after entering " + "safemode 2nd time", fsn.isPopulatingReplQueues()); }
@Test /** * Test that isInStartupSafemode returns true only during startup safemode * and not also during low-resource safemode */ public void testStartupSafemode() throws IOException { Configuration conf = new Configuration(); FSImage fsImage = Mockito.mock(FSImage.class); FSEditLog fsEditLog = Mockito.mock(FSEditLog.class); Mockito.when(fsImage.getEditLog()).thenReturn(fsEditLog); FSNamesystem fsn = new FSNamesystem(conf, fsImage); fsn.leaveSafeMode(); assertTrue("After leaving safemode FSNamesystem.isInStartupSafeMode still " + "returned true", !fsn.isInStartupSafeMode()); assertTrue("After leaving safemode FSNamesystem.isInSafeMode still returned" + " true", !fsn.isInSafeMode()); fsn.enterSafeMode(true); assertTrue("After entering safemode due to low resources FSNamesystem." + "isInStartupSafeMode still returned true", !fsn.isInStartupSafeMode()); assertTrue("After entering safemode due to low resources FSNamesystem." + "isInSafeMode still returned false", fsn.isInSafeMode()); }
FSNamesystem.getNamespaceDirs(conf); final Collection<URI> editsDirs = FSNamesystem.getNamespaceEditsDirs(conf); final Collection<URI> requiredEditsDirs = FSNamesystem.getRequiredNamespaceEditsDirs(conf); final Collection<URI> sharedEditsDirs = FSNamesystem.getSharedEditsDirs(conf);
long[] getStats() throws IOException { checkSuperuserPrivilege(); synchronized(heartbeats) { return new long[] {getCapacityTotal(), getCapacityUsed(), getCapacityRemaining()}; } }
Collection<URI> nameDirsToFormat = FSNamesystem.getNamespaceDirs(conf); List<URI> sharedDirs = FSNamesystem.getSharedEditsDirs(conf); List<URI> dirsToPrompt = new ArrayList<URI>(); dirsToPrompt.addAll(nameDirsToFormat); dirsToPrompt.addAll(sharedDirs); List<URI> editDirsToFormat = FSNamesystem.getNamespaceEditsDirs(conf); FSNamesystem fsn = new FSNamesystem(conf, fsImage); fsImage.getEditLog().initJournalsForWrite();
List<URI> sharedDirs = getSharedEditsDirs(conf); for (URI dir : getStorageDirs(conf, DFS_NAMENODE_EDITS_DIR_KEY)) { if (!editsDirs.add(dir)) { LOG.warn("Edits URI " + dir + " listed multiple times in " + return Lists.newArrayList(getNamespaceDirs(conf)); } else { return Lists.newArrayList(editsDirs);
@Override // ClientProtocol public LocatedBlocks getBlockLocations(String src, long offset, long length) throws IOException { checkNNStartup(); metrics.incrGetBlockLocations(); return namesystem.getBlockLocations(getClientMachine(), src, offset, length); }
@Test(timeout = 30000) public void testResolveReservedPath() throws IOException { FSNamesystem fsn = setupFileSystem(); FSEditLog editlog = fsn.getEditLog(); fsn.getBlockLocations("dummy", RESERVED_PATH, 0, 1024); verify(editlog).logTimes(eq(FILE_PATH), anyLong(), anyLong()); fsn.close(); }
public void run() { thr = Thread.currentThread(); PermissionStatus p = namesystem.createFsOwnerPermissions( new FsPermission((short)0777)); int i = 0; while (!stopped) { try { String dirname = "/thr-" + thr.getId() + "-dir-" + i; namesystem.mkdirs(dirname, p, true); namesystem.delete(dirname, true); } catch (SafeModeException sme) { // This is OK - the tests will bring NN in and out of safemode } catch (Throwable e) { LOG.warn("Got error in transaction thread", e); caught.compareAndSet(null, e); break; } i++; } }