synchronized List<Block> invalidateWork(final DatanodeDescriptor dn) { final long delay = getInvalidationDelay(); if (delay > 0) { BlockManager.LOG .debug("Block deletion is delayed during NameNode startup. " + "The deletion will start after {} ms.", delay); return null; } int remainingLimit = blockInvalidateLimit; final List<Block> toInvalidate = new ArrayList<>(); if (nodeToBlocks.get(dn) != null) { remainingLimit = getBlocksToInvalidateByLimit(nodeToBlocks.get(dn), toInvalidate, numBlocks, remainingLimit); } if ((remainingLimit > 0) && (nodeToECBlocks.get(dn) != null)) { getBlocksToInvalidateByLimit(nodeToECBlocks.get(dn), toInvalidate, numECBlocks, remainingLimit); } if (toInvalidate.size() > 0) { if (getBlockSetsSize(dn) == 0) { remove(dn); } dn.addBlocksToBeInvalidated(toInvalidate); } return toInvalidate; }
/** Used by metrics */ public long getPendingDeletionBlocksCount() { return invalidateBlocks.numBlocks(); } /** Used by metrics */
/** * Adds block to list of blocks which will be invalidated on specified * datanode and log the operation */ void addToInvalidates(final Block block, final DatanodeInfo datanode) { if (!isPopulatingReplQueues()) { return; } invalidateBlocks.add(block, datanode, true); }
/** Print the contents to out. */ synchronized void dump(final PrintWriter out) { final int size = nodeToBlocks.values().size() + nodeToECBlocks.values().size(); out.println("Metasave: Blocks " + numBlocks() + " waiting deletion from " + size + " datanodes."); if (size == 0) { return; } dumpBlockSet(nodeToBlocks, out); dumpBlockSet(nodeToECBlocks, out); }
LOG.warn("DataNode {} cannot be found with UUID {}" + ", removing block invalidation work.", dn, dn.getDatanodeUuid()); invalidateBlocks.remove(dn); return 0; toInvalidate = invalidateBlocks.invalidateWork(dnDescriptor);
synchronized List<Block> invalidateWork(final DatanodeDescriptor dn) { final long delay = getInvalidationDelay(); if (delay > 0) { if (BlockManager.LOG.isDebugEnabled()) { BlockManager.LOG .debug("Block deletion is delayed during NameNode startup. " + "The deletion will start after " + delay + " ms."); } return null; } final LightWeightHashSet<Block> set = node2blocks.get(dn); if (set == null) { return null; } // # blocks that can be sent in one message is limited final int limit = blockInvalidateLimit; final List<Block> toInvalidate = set.pollN(limit); // If we send everything in this message, remove this node entry if (set.isEmpty()) { remove(dn); } dn.addBlocksToBeInvalidated(toInvalidate); numBlocks -= toInvalidate.size(); return toInvalidate; }
@VisibleForTesting public boolean containsInvalidateBlock(final DatanodeInfo dn, final Block block) { return invalidateBlocks.contains(dn, block); }
InvalidateBlocks(final int blockInvalidateLimit, long pendingPeriodInMs, final BlockIdManager blockIdManager) { this.blockInvalidateLimit = blockInvalidateLimit; this.pendingPeriodInMs = pendingPeriodInMs; this.blockIdManager = blockIdManager; printBlockDeletionTime(BlockManager.LOG); }
/** * Clear all queues that hold decisions previously made by * this NameNode. */ public void clearQueues() { neededReconstruction.clear(); pendingReconstruction.clear(); excessRedundancyMap.clear(); invalidateBlocks.clear(); datanodeManager.clearPendingQueues(); postponedMisreplicatedBlocks.clear(); };
/** * Schedule blocks for deletion at datanodes * @param nodesToProcess number of datanodes to schedule deletion work * @return total number of block for deletion */ int computeInvalidateWork(int nodesToProcess) { final List<DatanodeInfo> nodes = invalidateBlocks.getDatanodes(); Collections.shuffle(nodes); nodesToProcess = Math.min(nodes.size(), nodesToProcess); int blockCnt = 0; for (DatanodeInfo dnInfo : nodes) { int blocks = invalidateWorkForOneNode(dnInfo); if (blocks > 0) { blockCnt += blocks; if (--nodesToProcess == 0) { break; } } } return blockCnt; }
invalidateBlocks.dump(out);
DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY, DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 1000L; invalidateBlocks = new InvalidateBlocks( datanodeManager.getBlockInvalidateLimit(), startupDelayBlockDeletionInMs,
"invalidateBlocks"); InvalidateBlocks mockIb = Mockito.spy(invalidateBlocks); Mockito.doReturn(1L).when(mockIb).getInvalidationDelay(); Whitebox.setInternalState(cluster.getNamesystem().getBlockManager(), "invalidateBlocks", mockIb);
LOG.warn("DataNode " + dn + " cannot be found with UUID " + dn.getDatanodeUuid() + ", removing block invalidation work."); invalidateBlocks.remove(dn); return 0; toInvalidate = invalidateBlocks.invalidateWork(dnDescriptor);
synchronized List<Block> invalidateWork(final DatanodeDescriptor dn) { final long delay = getInvalidationDelay(); if (delay > 0) { if (BlockManager.LOG.isDebugEnabled()) { BlockManager.LOG .debug("Block deletion is delayed during NameNode startup. " + "The deletion will start after " + delay + " ms."); } return null; } final LightWeightHashSet<Block> set = node2blocks.get(dn); if (set == null) { return null; } // # blocks that can be sent in one message is limited final int limit = blockInvalidateLimit; final List<Block> toInvalidate = set.pollN(limit); // If we send everything in this message, remove this node entry if (set.isEmpty()) { remove(dn); } dn.addBlocksToBeInvalidated(toInvalidate); numBlocks -= toInvalidate.size(); return toInvalidate; }
/** * Get all valid locations of the block */ private List<DatanodeStorageInfo> getValidLocations(BlockInfo block) { final List<DatanodeStorageInfo> locations = new ArrayList<DatanodeStorageInfo>(blocksMap.numNodes(block)); for(DatanodeStorageInfo storage : blocksMap.getStorages(block)) { // filter invalidate replicas Block b = getBlockOnStorage(block, storage); if(b != null && !invalidateBlocks.contains(storage.getDatanodeDescriptor(), b)) { locations.add(storage); } } return locations; }
InvalidateBlocks(final int blockInvalidateLimit, long pendingPeriodInMs) { this.blockInvalidateLimit = blockInvalidateLimit; this.pendingPeriodInMs = pendingPeriodInMs; printBlockDeletionTime(BlockManager.LOG); }
/** * Clear all queues that hold decisions previously made by * this NameNode. */ public void clearQueues() { neededReplications.clear(); pendingReplications.clear(); excessReplicateMap.clear(); invalidateBlocks.clear(); datanodeManager.clearPendingQueues(); postponedMisreplicatedBlocks.clear(); };
/** * Schedule blocks for deletion at datanodes * @param nodesToProcess number of datanodes to schedule deletion work * @return total number of block for deletion */ int computeInvalidateWork(int nodesToProcess) { final List<DatanodeInfo> nodes = invalidateBlocks.getDatanodes(); Collections.shuffle(nodes); nodesToProcess = Math.min(nodes.size(), nodesToProcess); int blockCnt = 0; for (DatanodeInfo dnInfo : nodes) { int blocks = invalidateWorkForOneNode(dnInfo); if (blocks > 0) { blockCnt += blocks; if (--nodesToProcess == 0) { break; } } } return blockCnt; }
invalidateBlocks.dump(out);