/** * Adds block to list of blocks which will be invalidated on specified * datanode and log the operation */ void addToInvalidates(final Block block, final DatanodeInfo datanode) { if (!isPopulatingReplQueues()) { return; } invalidateBlocks.add(block, datanode, true); }
/** * Check if the cluster now consists of multiple racks. If it does, and this * is the first time it's consisted of multiple racks, then process blocks * that may now be misreplicated. * * @param node DN which caused cluster to become multi-rack. Used for logging. */ @VisibleForTesting void checkIfClusterIsNowMultiRack(DatanodeDescriptor node) { if (!hasClusterEverBeenMultiRack && networktopology.getNumOfRacks() > 1) { String message = "DN " + node + " joining cluster has expanded a formerly " + "single-rack cluster to be multi-rack. "; if (blockManager.isPopulatingReplQueues()) { message += "Re-checking all blocks for replication, since they should " + "now be replicated cross-rack"; LOG.info(message); } else { message += "Not checking for mis-replicated blocks because this NN is " + "not yet processing repl queues."; LOG.debug(message); } hasClusterEverBeenMultiRack = true; if (blockManager.isPopulatingReplQueues()) { blockManager.processMisReplicatedBlocks(); } } }
@Override public void run() { while (namesystem.isRunning()) { try { // Check storage efficiency only when active NN is out of safe mode. if (isPopulatingReplQueues()) { scanAndCompactStorages(); } Thread.sleep(storageInfoDefragmentInterval); } catch (Throwable t) { if (!namesystem.isRunning()) { LOG.info("Stopping thread."); if (!(t instanceof InterruptedException)) { LOG.info("Received an exception while shutting down.", t); } break; } else if (!checkNSRunning && t instanceof InterruptedException) { LOG.info("Stopping for testing."); break; } LOG.error("Thread received Runtime exception.", t); terminate(1, t); } } }
/** Check if we are ready to initialize replication queues. */ private void initializeReplQueuesIfNecessary() { assert namesystem.hasWriteLock(); // Whether it has reached the threshold for initializing replication queues. boolean canInitializeReplQueues = blockManager.shouldPopulateReplQueues() && blockSafe >= blockReplQueueThreshold; if (canInitializeReplQueues && !blockManager.isPopulatingReplQueues() && !haEnabled) { blockManager.initializeReplQueues(); } }
/** * Adds block to list of blocks which will be invalidated on all its * datanodes. */ private void addToInvalidates(BlockInfo storedBlock) { if (!isPopulatingReplQueues()) { return; } StringBuilder datanodes = blockLog.isDebugEnabled() ? new StringBuilder() : null; for (DatanodeStorageInfo storage : blocksMap.getStorages(storedBlock)) { if (storage.getState() != State.NORMAL) { continue; } final DatanodeDescriptor node = storage.getDatanodeDescriptor(); final Block b = getBlockOnStorage(storedBlock, storage); if (b != null) { invalidateBlocks.add(b, node, false); if (datanodes != null) { datanodes.append(node).append(" "); } } } if (datanodes != null && datanodes.length() != 0) { blockLog.debug("BLOCK* addToInvalidates: {} {}", storedBlock, datanodes); } }
@Override public void run() { while (namesystem.isRunning()) { try { // Process recovery work only when active NN is out of safe mode. if (isPopulatingReplQueues()) { computeDatanodeWork(); processPendingReconstructions(); rescanPostponedMisreplicatedBlocks(); } TimeUnit.MILLISECONDS.sleep(redundancyRecheckIntervalMs); } catch (Throwable t) { if (!namesystem.isRunning()) { LOG.info("Stopping RedundancyMonitor."); if (!(t instanceof InterruptedException)) { LOG.info("RedundancyMonitor received an exception" + " while shutting down.", t); } break; } else if (!checkNSRunning && t instanceof InterruptedException) { LOG.info("Stopping RedundancyMonitor for testing."); break; } LOG.error("RedundancyMonitor thread received Runtime exception. ", t); terminate(1, t); } } } }
/** * On putting the node in service, check if the node has excess replicas. * If there are any excess replicas, call processExtraRedundancyBlock(). * Process extra redundancy blocks only when active NN is out of safe mode. */ void processExtraRedundancyBlocksOnInService( final DatanodeDescriptor srcNode) { if (!isPopulatingReplQueues()) { return; } final Iterator<BlockInfo> it = srcNode.getBlockIterator(); int numExtraRedundancy = 0; while(it.hasNext()) { final BlockInfo block = it.next(); if (block.isDeleted()) { //Orphan block, will be handled eventually, skip continue; } int expectedReplication = this.getExpectedRedundancyNum(block); NumberReplicas num = countNodes(block); if (shouldProcessExtraRedundancy(num, expectedReplication)) { // extra redundancy block processExtraRedundancyBlock(block, (short) expectedReplication, null, null); numExtraRedundancy++; } } LOG.info("Invalidated {} extra redundancy blocks on {} after " + "it is in service", numExtraRedundancy, srcNode); }
try { checkOperation(OperationCategory.READ); if (!blockManager.isPopulatingReplQueues()) { throw new IOException("Cannot run listCorruptFileBlocks because " + "replication queues have not been initialized.");
if (!blockManager.isPopulatingReplQueues() && blockManager.shouldPopulateReplQueues()) { blockManager.initializeReplQueues();
if (!blockManager.neededReconstruction.contains(block) && blockManager.pendingReconstruction.getNumReplicas(block) == 0 && blockManager.isPopulatingReplQueues()) {
assert (storedBlock != null && namesystem.hasWriteLock()); if (!namesystem.isInStartupSafeMode() || isPopulatingReplQueues()) { addStoredBlock(storedBlock, reported, storageInfo, null, false); return;
nnResourceChecker = new NameNodeResourceChecker(conf); checkAvailableResources(); assert !blockManager.isPopulatingReplQueues(); StartupProgress prog = NameNode.getStartupProgress(); prog.beginPhase(Phase.SAFEMODE);
/** updates a block in needed reconstruction queue. */ private void updateNeededReconstructions(final BlockInfo block, final int curReplicasDelta, int expectedReplicasDelta) { namesystem.writeLock(); try { if (!isPopulatingReplQueues() || !block.isComplete()) { return; } NumberReplicas repl = countNodes(block); int pendingNum = pendingReconstruction.getNumReplicas(block); int curExpectedReplicas = getExpectedRedundancyNum(block); if (!hasEnoughEffectiveReplicas(block, repl, pendingNum)) { neededReconstruction.update(block, repl.liveReplicas() + pendingNum, repl.readOnlyReplicas(), repl.outOfServiceReplicas(), curExpectedReplicas, curReplicasDelta, expectedReplicasDelta); } else { int oldReplicas = repl.liveReplicas() + pendingNum - curReplicasDelta; int oldExpectedReplicas = curExpectedReplicas-expectedReplicasDelta; neededReconstruction.remove(block, oldReplicas, repl.readOnlyReplicas(), repl.outOfServiceReplicas(), oldExpectedReplicas); } } finally { namesystem.writeUnlock(); } }
if (!isPopulatingReplQueues()) { return storedBlock;
} else if (isPopulatingReplQueues()) {