/** * Add a block to insufficiently redundant queue according to its priority. * * @param block a low redundancy block * @param curReplicas current number of replicas of the block * @param outOfServiceReplicas the number of out-of-service replicas * @param expectedReplicas expected number of replicas of the block * @return true if the block was added to a queue. */ synchronized boolean add(BlockInfo block, int curReplicas, int readOnlyReplicas, int outOfServiceReplicas, int expectedReplicas) { final int priLevel = getPriority(block, curReplicas, readOnlyReplicas, outOfServiceReplicas, expectedReplicas); if(add(block, priLevel, expectedReplicas)) { NameNode.blockStateChangeLog.debug( "BLOCK* NameSystem.LowRedundancyBlock.add: {}" + " has only {} replicas and need {} replicas so is added to" + " neededReconstructions at priority level {}", block, curReplicas, expectedReplicas, priLevel); return true; } return false; }
public long getMissingBlocksCount() { // not locking return this.neededReconstruction.getCorruptBlockSize(); }
/** Used by metrics. */ public long getMissingBlocks() { return neededReconstruction.getCorruptBlocks(); }
int oldReplicas = curReplicas-curReplicasDelta; int oldExpectedReplicas = curExpectedReplicas-expectedReplicasDelta; int curPri = getPriority(block, curReplicas, readOnlyReplicas, outOfServiceReplicas, curExpectedReplicas); int oldPri = getPriority(block, oldReplicas, readOnlyReplicas, outOfServiceReplicas, oldExpectedReplicas); if(NameNode.stateChangeLog.isDebugEnabled()) { remove(block, oldPri, oldExpectedReplicas); if(add(block, curPri, curExpectedReplicas)) { NameNode.blockStateChangeLog.debug( "BLOCK* NameSystem.LowRedundancyBlock.update: {} has only {} "
+ neededReconstruction.getLowRedundancyBlockCount()); for (int i = 0; i < neededReconstruction.LEVEL; i++) { if (i != neededReconstruction.QUEUE_WITH_CORRUPT_BLOCKS) { for (Iterator<BlockInfo> it = neededReconstruction.iterator(i); it.hasNext();) { Block block = it.next(); neededReconstruction.getCorruptBlockSize()); for (Iterator<BlockInfo> it = neededReconstruction. iterator(neededReconstruction.QUEUE_WITH_CORRUPT_BLOCKS); it.hasNext();) { Block block = it.next();
blockManager.isNeededReconstructionForMaintenance(block, num); if (neededReconstruction) { if (!blockManager.neededReconstruction.contains(block) && blockManager.pendingReconstruction.getNumReplicas(block) == 0 && blockManager.isPopulatingReplQueues()) { blockManager.neededReconstruction.add(block, liveReplicas, num.readOnlyReplicas(), num.outOfServiceReplicas(),
/** * Check sufficient redundancy of the blocks in the collection. If any block * is needed reconstruction, insert it into the reconstruction queue. * Otherwise, if the block is more than the expected replication factor, * process it as an extra redundancy block. */ public void checkRedundancy(BlockCollection bc) { for (BlockInfo block : bc.getBlocks()) { short expected = getExpectedRedundancyNum(block); final NumberReplicas n = countNodes(block); final int pending = pendingReconstruction.getNumReplicas(block); if (!hasEnoughEffectiveReplicas(block, n, pending)) { neededReconstruction.add(block, n.liveReplicas() + pending, n.readOnlyReplicas(), n.outOfServiceReplicas(), expected); } else if (shouldProcessExtraRedundancy(n, expected)) { processExtraRedundancyBlock(block, expected, null, null); } } }
/** * Clear all queues that hold decisions previously made by * this NameNode. */ public void clearQueues() { neededReconstruction.clear(); pendingReconstruction.clear(); excessRedundancyMap.clear(); invalidateBlocks.clear(); datanodeManager.clearPendingQueues(); postponedMisreplicatedBlocks.clear(); };
/** Used by metrics. */ public long getMissingReplicationOneBlocks() { return neededReconstruction.getCorruptReplicationOneBlocks(); }
/** Used by metrics. */ public long getMissingECBlockGroups() { return neededReconstruction.getCorruptECBlockGroups(); }
public long getMissingReplOneBlocksCount() { // not locking return this.neededReconstruction.getCorruptReplicationOneBlockSize(); }
boolean remove(BlockInfo block, int priLevel, int oldExpectedReplicas) { if(priLevel >= 0 && priLevel < LEVEL && priorityQueues.get(priLevel).remove(block)) { NameNode.blockStateChangeLog.debug( "BLOCK* NameSystem.LowRedundancyBlock.remove: Removing block {}" + " from priority queue {}", block, priLevel); decrementBlockStat(block, priLevel, oldExpectedReplicas); return true; } else { // Try to remove the block from all queues if the block was // not found in the queue for the given priority level. for (int i = 0; i < LEVEL; i++) { if (i != priLevel && priorityQueues.get(i).remove(block)) { NameNode.blockStateChangeLog.debug( "BLOCK* NameSystem.LowRedundancyBlock.remove: Removing block" + " {} from priority queue {}", block, i); decrementBlockStat(block, i, oldExpectedReplicas); return true; } } } return false; }
/** * Scan blocks in {@link #neededReconstruction} and assign reconstruction * (replication or erasure coding) work to data-nodes they belong to. * * The number of process blocks equals either twice the number of live * data-nodes or the number of low redundancy blocks whichever is less. * * @return number of blocks scheduled for reconstruction during this * iteration. */ int computeBlockReconstructionWork(int blocksToProcess) { List<List<BlockInfo>> blocksToReconstruct = null; namesystem.writeLock(); try { // Choose the blocks to be reconstructed blocksToReconstruct = neededReconstruction .chooseLowRedundancyBlocks(blocksToProcess); } finally { namesystem.writeUnlock(); } return computeReconstructionWorkForBlocks(blocksToReconstruct); }
neededReconstruction.add(bi, num.liveReplicas(), num.readOnlyReplicas(), num.outOfServiceReplicas(), getExpectedRedundancyNum(bi));
/** * For each block in the name-node verify whether it belongs to any file, * extra or low redundancy. Place it into the respective queue. */ public void processMisReplicatedBlocks() { assert namesystem.hasWriteLock(); stopReconstructionInitializer(); neededReconstruction.clear(); reconstructionQueuesInitializer = new Daemon() { @Override public void run() { try { processMisReplicatesAsync(); } catch (InterruptedException ie) { LOG.info("Interrupted while processing reconstruction queues."); } catch (Exception e) { LOG.error("Error while processing reconstruction queues async", e); } } }; reconstructionQueuesInitializer .setName("Reconstruction Queue Initializer"); reconstructionQueuesInitializer.start(); }
/** Return the number of corrupt blocks with replication factor 1 */ long getCorruptReplicationOneBlockSize() { return getCorruptReplicationOneBlocks(); }
/** * Return low redundancy striped blocks excluding corrupt blocks. */ long getLowRedundancyECBlockGroups() { return lowRedundancyECBlockGroups.longValue() - getCorruptECBlockGroups(); }
if (neededReconstruction.add(block, numCurrentReplica, num.readOnlyReplicas(), num.outOfServiceReplicas(), expectedRedundancy)) {
/** * Return under replicated block count excluding corrupt replicas. */ long getLowRedundancyBlocks() { return lowRedundancyBlocks.longValue() - getCorruptBlocks(); }