private void processChosenExcessRedundancy( final Collection<DatanodeStorageInfo> nonExcess, final DatanodeStorageInfo chosen, BlockInfo storedBlock) { nonExcess.remove(chosen); excessRedundancyMap.add(chosen.getDatanodeDescriptor(), storedBlock); // // The 'excessblocks' tracks blocks until we get confirmation // that the datanode has deleted them; the only way we remove them // is when we get a "removeBlock" message. // // The 'invalidate' list is used to inform the datanode the block // should be deleted. Items are removed from the invalidate list // upon giving instructions to the datanodes. // final Block blockToInvalidate = getBlockOnStorage(storedBlock, chosen); addToInvalidates(blockToInvalidate, chosen.getDatanodeDescriptor()); blockLog.debug("BLOCK* chooseExcessRedundancies: " + "({}, {}) is added to invalidated blocks set", chosen, storedBlock); }
public void removeBlock(BlockInfo block) { assert namesystem.hasWriteLock(); // No need to ACK blocks that are being removed entirely // from the namespace, since the removal of the associated // file already removes them from the block map below. block.setNumBytes(BlockCommand.NO_ACK); addToInvalidates(block); removeBlockFromMap(block); // Remove the block from pendingReconstruction and neededReconstruction pendingReconstruction.remove(block); neededReconstruction.remove(block, LowRedundancyBlocks.LEVEL); postponedMisreplicatedBlocks.remove(block); }
addToInvalidates(b.getCorrupted(), dn); removeStoredBlock(b.getStored(), node); blockLog.debug("BLOCK* invalidateBlocks: {} on {} listed for deletion.",
addToInvalidates(new Block(block), node); return;
if (block.isDeleted()) { addToInvalidates(block); return MisReplicationResult.INVALID;
private void processChosenExcessReplica( final Collection<DatanodeStorageInfo> nonExcess, final DatanodeStorageInfo chosen, Block b) { nonExcess.remove(chosen); addToExcessReplicate(chosen.getDatanodeDescriptor(), b); // // The 'excessblocks' tracks blocks until we get confirmation // that the datanode has deleted them; the only way we remove them // is when we get a "removeBlock" message. // // The 'invalidate' list is used to inform the datanode the block // should be deleted. Items are removed from the invalidate list // upon giving instructions to the datanodes. // addToInvalidates(b, chosen.getDatanodeDescriptor()); blockLog.debug("BLOCK* chooseExcessReplicates: " + "({}, {}) is added to invalidated blocks set", chosen, b); }
blockLog.debug("BLOCK markBlockAsCorrupt: {} cannot be marked as" + " corrupt as it does not belong to any file", b); addToInvalidates(b.getCorrupted(), node); return;
blockLog.info("BLOCK* addBlock: block {} on node {} size {} does not " + "belong to any file", b, node, b.getNumBytes()); addToInvalidates(b, node);
private void processChosenExcessReplica( final Collection<DatanodeStorageInfo> nonExcess, final DatanodeStorageInfo chosen, Block b) { nonExcess.remove(chosen); addToExcessReplicate(chosen.getDatanodeDescriptor(), b); // // The 'excessblocks' tracks blocks until we get confirmation // that the datanode has deleted them; the only way we remove them // is when we get a "removeBlock" message. // // The 'invalidate' list is used to inform the datanode the block // should be deleted. Items are removed from the invalidate list // upon giving instructions to the datanodes. // addToInvalidates(b, chosen.getDatanodeDescriptor()); blockLog.debug("BLOCK* chooseExcessReplicates: " + "({}, {}) is added to invalidated blocks set", chosen, b); }
public void removeBlock(Block block) { assert namesystem.hasWriteLock(); // No need to ACK blocks that are being removed entirely // from the namespace, since the removal of the associated // file already removes them from the block map below. block.setNumBytes(BlockCommand.NO_ACK); addToInvalidates(block); removeBlockFromMap(block); // Remove the block from pendingReplications and neededReplications pendingReplications.remove(block); neededReplications.remove(block, UnderReplicatedBlocks.LEVEL); postponedMisreplicatedBlocks.remove(block); }
public void removeBlock(Block block) { assert namesystem.hasWriteLock(); // No need to ACK blocks that are being removed entirely // from the namespace, since the removal of the associated // file already removes them from the block map below. block.setNumBytes(BlockCommand.NO_ACK); addToInvalidates(block); removeBlockFromMap(block); // Remove the block from pendingReplications and neededReplications pendingReplications.remove(block); neededReplications.remove(block, UnderReplicatedBlocks.LEVEL); postponedMisreplicatedBlocks.remove(block); }
} else if (nr.liveReplicas() >= 1) { addToInvalidates(b.corrupted, dn); removeStoredBlock(b.stored, node); blockLog.debug("BLOCK* invalidateBlocks: {} on {} listed for deletion.",
} else if (nr.liveReplicas() >= 1) { addToInvalidates(b.corrupted, dn); removeStoredBlock(b.stored, node); blockLog.debug("BLOCK* invalidateBlocks: {} on {} listed for deletion.",
if (bc == null) { addToInvalidates(block); return MisReplicationResult.INVALID;
blockLog.info("BLOCK markBlockAsCorrupt: {} cannot be marked as" + " corrupt as it does not belong to any file", b); addToInvalidates(b.corrupted, node); return;
DatanodeDescriptor dn = bm.blocksMap.getStorages(b.getLocalBlock()) .iterator().next().getDatanodeDescriptor(); bm.addToInvalidates(b.getLocalBlock(), dn); Thread.sleep(5000); bm.blocksMap.removeNode(b.getLocalBlock(), dn);
/** * Reformatted DataNodes will replace the original UUID in the * {@link DatanodeManager#datanodeMap}. This tests if block * invalidation work on the original DataNode can be skipped. */ @Test(timeout=120000) public void testDatanodeReformat() throws Exception { namesystem.writeLock(); try { // Change the datanode UUID to emulate a reformat String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnr = cluster.getDataNode(nodes[0].getIpcPort()) .getDNRegistrationForBP(poolId); dnr = new DatanodeRegistration(UUID.randomUUID().toString(), dnr); cluster.stopDataNode(nodes[0].getXferAddr()); Block block = new Block(0, 0, GenerationStamp.LAST_RESERVED_STAMP); bm.addToInvalidates(block, nodes[0]); bm.getDatanodeManager().registerDatanode(dnr); // Since UUID has changed, the invalidation work should be skipped assertEquals(0, bm.computeInvalidateWork(1)); assertEquals(0, bm.getPendingDeletionBlocksCount()); } finally { namesystem.writeUnlock(); } }
Block block = new Block(i*(blockInvalidateLimit+1)+j, 0, GenerationStamp.LAST_RESERVED_STAMP); bm.addToInvalidates(block, nodes[i]);