/** * Clear all queues that hold decisions previously made by * this NameNode. */ public void clearQueues() { neededReconstruction.clear(); pendingReconstruction.clear(); excessRedundancyMap.clear(); invalidateBlocks.clear(); datanodeManager.clearPendingQueues(); postponedMisreplicatedBlocks.clear(); };
/** Used by metrics. */ public long getNumTimedOutPendingReconstructions() { return pendingReconstruction.getNumTimedOuts(); }
/** * If IBR is not sent from expected locations yet, add the datanodes to * pendingReconstruction in order to keep RedundancyMonitor from scheduling * the block. */ public void addExpectedReplicasToPending(BlockInfo blk) { if (!blk.isStriped()) { DatanodeStorageInfo[] expectedStorages = blk.getUnderConstructionFeature().getExpectedStorageLocations(); if (expectedStorages.length - blk.numNodes() > 0) { ArrayList<DatanodeDescriptor> pendingNodes = new ArrayList<>(); for (DatanodeStorageInfo storage : expectedStorages) { DatanodeDescriptor dnd = storage.getDatanodeDescriptor(); if (blk.findStorageInfo(dnd) == null) { pendingNodes.add(dnd); } } pendingReconstruction.increment(blk, pendingNodes.toArray(new DatanodeDescriptor[pendingNodes.size()])); } } }
int pendingNum = pendingReconstruction.getNumReplicas(block); if (hasEnoughEffectiveReplicas(block, numReplicas, pendingNum)) { neededReconstruction.remove(block, priority);
final short requiredRedundancy = getExpectedLiveRedundancyNum(block, numReplicas); final int pendingNum = pendingReconstruction.getNumReplicas(block); if (hasEnoughEffectiveReplicas(block, numReplicas, pendingNum)) { neededReconstruction.remove(block, priority); pendingReconstruction.increment(block, DatanodeStorageInfo.toDatanodeDescriptors(targets)); blockLog.debug("BLOCK* block {} is moved from neededReconstruction to "
void updateState() { pendingReconstructionBlocksCount = pendingReconstruction.size(); lowRedundancyBlocksCount = neededReconstruction.size(); corruptReplicaBlocksCount = corruptReplicas.size(); }
public void removeBlock(BlockInfo block) { assert namesystem.hasWriteLock(); // No need to ACK blocks that are being removed entirely // from the namespace, since the removal of the associated // file already removes them from the block map below. block.setNumBytes(BlockCommand.NO_ACK); addToInvalidates(block); removeBlockFromMap(block); // Remove the block from pendingReconstruction and neededReconstruction pendingReconstruction.remove(block); neededReconstruction.remove(block, LowRedundancyBlocks.LEVEL); postponedMisreplicatedBlocks.remove(block); }
datanodeManager.getHost2DatanodeMap()); storagePolicySuite = BlockStoragePolicySuite.createDefaultSuite(); pendingReconstruction = new PendingReconstructionBlocks(conf.getInt( DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY, DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_DEFAULT)
if (storedBlock != null && block.getGenerationStamp() == storedBlock.getGenerationStamp()) { if (pendingReconstruction.decrement(storedBlock, node)) { NameNode.getNameNodeMetrics().incSuccessfulReReplications();
pendingReconstruction.metaSave(out);
BlockInfo[] timedOutItems = pendingReconstruction.getTimedOutBlocks(); if (timedOutItems != null) { namesystem.writeLock();
/** * Check sufficient redundancy of the blocks in the collection. If any block * is needed reconstruction, insert it into the reconstruction queue. * Otherwise, if the block is more than the expected replication factor, * process it as an extra redundancy block. */ public void checkRedundancy(BlockCollection bc) { for (BlockInfo block : bc.getBlocks()) { short expected = getExpectedRedundancyNum(block); final NumberReplicas n = countNodes(block); final int pending = pendingReconstruction.getNumReplicas(block); if (!hasEnoughEffectiveReplicas(block, n, pending)) { neededReconstruction.add(block, n.liveReplicas() + pending, n.readOnlyReplicas(), n.outOfServiceReplicas(), expected); } else if (shouldProcessExtraRedundancy(n, expected)) { processExtraRedundancyBlock(block, expected, null, null); } } }
neededReconstruction.size(), pendingReconstruction.size());
replicas.readOnlyReplicas(), replicas.outOfServiceReplicas(), getExpectedRedundancyNum(lastBlock)); pendingReconstruction.remove(lastBlock);
if (neededReconstruction) { if (!blockManager.neededReconstruction.contains(block) && blockManager.pendingReconstruction.getNumReplicas(block) == 0 && blockManager.isPopulatingReplQueues()) {
/** updates a block in needed reconstruction queue. */ private void updateNeededReconstructions(final BlockInfo block, final int curReplicasDelta, int expectedReplicasDelta) { namesystem.writeLock(); try { if (!isPopulatingReplQueues() || !block.isComplete()) { return; } NumberReplicas repl = countNodes(block); int pendingNum = pendingReconstruction.getNumReplicas(block); int curExpectedReplicas = getExpectedRedundancyNum(block); if (!hasEnoughEffectiveReplicas(block, repl, pendingNum)) { neededReconstruction.update(block, repl.liveReplicas() + pendingNum, repl.readOnlyReplicas(), repl.outOfServiceReplicas(), curExpectedReplicas, curReplicasDelta, expectedReplicasDelta); } else { int oldReplicas = repl.liveReplicas() + pendingNum - curReplicasDelta; int oldExpectedReplicas = curExpectedReplicas-expectedReplicasDelta; neededReconstruction.remove(block, oldReplicas, repl.readOnlyReplicas(), repl.outOfServiceReplicas(), oldExpectedReplicas); } } finally { namesystem.writeUnlock(); } }
int pendingNum = pendingReconstruction.getNumReplicas(storedBlock); int numCurrentReplica = numLiveReplicas + pendingNum;