public BlockInfo(Block blk, short size) { super(blk); this.storages = new DatanodeStorageInfo[size]; this.bcId = INVALID_INODE_ID; this.replication = isStriped() ? 0 : size; }
/** * Construct an entry for blocksmap * @param size the block's replication factor, or the total number of blocks * in the block group */ public BlockInfo(short size) { this.storages = new DatanodeStorageInfo[size]; this.bcId = INVALID_INODE_ID; this.replication = isStriped() ? 0 : size; }
private void incrementBlockStat(BlockInfo block) { if (block.isStriped()) { totalECBlockGroups.increment(); } else { totalReplicatedBlocks.increment(); } }
private Block getBlockOnStorage(BlockInfo storedBlock, DatanodeStorageInfo storage) { return storedBlock.isStriped() ? ((BlockInfoStriped) storedBlock).getBlockOnStorage(storage) : storedBlock; }
@Override // BlockCollection public void setBlock(int index, BlockInfo blk) { Preconditions.checkArgument(blk.isStriped() == this.isStriped()); this.blocks[index] = blk; }
/** * Remove Block from CorruptBlocksMap. * @param blk Block to be removed */ void removeFromCorruptReplicasMap(BlockInfo blk) { if (corruptReplicasMap != null) { Map<DatanodeDescriptor, Reason> value = corruptReplicasMap.remove(blk); if (value != null) { decrementBlockStat(blk.isStriped()); } } }
private void incrementBlockStat(BlockInfo blockInfo, int priLevel, int expectedReplicas) { if (blockInfo.isStriped()) { lowRedundancyECBlockGroups.increment(); if (priLevel == QUEUE_WITH_CORRUPT_BLOCKS) { corruptECBlockGroups.increment(); } if (priLevel == QUEUE_HIGHEST_PRIORITY) { highestPriorityLowRedundancyECBlocks.increment(); } } else { lowRedundancyBlocks.increment(); if (priLevel == QUEUE_WITH_CORRUPT_BLOCKS) { corruptBlocks.increment(); if (expectedReplicas == 1) { corruptReplicationOneBlocks.increment(); } } if (priLevel == QUEUE_HIGHEST_PRIORITY) { highestPriorityLowRedundancyReplicatedBlocks.increment(); } } }
boolean removeFromCorruptReplicasMap( BlockInfo blk, DatanodeDescriptor datanode, Reason reason) { Map <DatanodeDescriptor, Reason> datanodes = corruptReplicasMap.get(blk); if (datanodes == null) { return false; } // if reasons can be compared but don't match, return false. Reason storedReason = datanodes.get(datanode); if (reason != Reason.ANY && storedReason != null && reason != storedReason) { return false; } if (datanodes.remove(datanode) != null) { // remove the replicas if (datanodes.isEmpty()) { // remove the block if there is no more corrupted replicas corruptReplicasMap.remove(blk); decrementBlockStat(blk.isStriped()); } return true; } return false; }
private short getMinMaintenanceStorageNum(BlockInfo block) { if (block.isStriped()) { return ((BlockInfoStriped) block).getRealDataBlockNum(); } else { return (short) Math.min(minReplicationToBeInMaintenance, block.getReplication()); } }
public short getExpectedRedundancyNum(BlockInfo block) { return block.isStriped() ? ((BlockInfoStriped) block).getRealTotalBlockNum() : block.getReplication(); }
/** * add a block to the block list */ void addBlock(BlockInfo newblock) { Preconditions.checkArgument(newblock.isStriped() == this.isStriped()); if (this.blocks.length == 0) { this.setBlocks(new BlockInfo[]{newblock}); } else { int size = this.blocks.length; BlockInfo[] newlist = new BlockInfo[size + 1]; System.arraycopy(this.blocks, 0, newlist, 0, size); newlist[size] = newblock; this.setBlocks(newlist); } }
/** * Do some check when adding a block to blocksmap. * For HDFS-7994 to check whether then block is a NonEcBlockUsingStripedID. * */ public BlockInfo addBlockCollectionWithCheck( BlockInfo block, BlockCollection bc) { if (!hasNonEcBlockUsingStripedID && !block.isStriped() && BlockIdManager.isStripedBlockID(block.getBlockId())) { hasNonEcBlockUsingStripedID = true; } return addBlockCollection(block, bc); }
private void decrementBlockStat(BlockInfo block) { if (block.isStriped()) { totalECBlockGroups.decrement(); assert totalECBlockGroups.longValue() >= 0 : "Total number of ec block groups should be non-negative"; } else { totalReplicatedBlocks.decrement(); assert totalReplicatedBlocks.longValue() >= 0 : "Total number of replicated blocks should be non-negative"; } }
public static LocatedBlock newLocatedBlock(ExtendedBlock eb, BlockInfo info, DatanodeStorageInfo[] locs, long offset) throws IOException { final LocatedBlock lb; if (info.isStriped()) { lb = newLocatedStripedBlock(eb, locs, info.getUnderConstructionFeature().getBlockIndices(), offset, false); } else { lb = newLocatedBlock(eb, locs, offset, false); } return lb; }
public final QuotaCounts storagespaceConsumedStriped() { QuotaCounts counts = new QuotaCounts.Builder().build(); for (BlockInfo b : blocks) { Preconditions.checkState(b.isStriped()); long blockSize = b.isComplete() ? ((BlockInfoStriped)b).spaceConsumed() : getPreferredBlockSize() * ((BlockInfoStriped)b).getTotalBlockNum(); counts.addStorageSpace(blockSize); } return counts; }
/** Return the priority of a block * @param curReplicas current number of replicas of the block * @param expectedReplicas expected number of replicas of the block * @return the priority for the blocks, between 0 and ({@link #LEVEL}-1) */ private int getPriority(BlockInfo block, int curReplicas, int readOnlyReplicas, int outOfServiceReplicas, int expectedReplicas) { assert curReplicas >= 0 : "Negative replicas!"; if (curReplicas >= expectedReplicas) { // Block has enough copies, but not enough racks return QUEUE_REPLICAS_BADLY_DISTRIBUTED; } if (block.isStriped()) { BlockInfoStriped sblk = (BlockInfoStriped) block; return getPriorityStriped(curReplicas, outOfServiceReplicas, sblk.getRealDataBlockNum(), sblk.getParityBlockNum()); } else { return getPriorityContiguous(curReplicas, readOnlyReplicas, outOfServiceReplicas, expectedReplicas); } }
NumberReplicas countNodes(BlockInfo b, boolean inStartupSafeMode) { NumberReplicas numberReplicas = new NumberReplicas(); Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(b); if (b.isStriped()) { countReplicasForStripedBlock(numberReplicas, (BlockInfoStriped) b, nodesCorrupt, inStartupSafeMode); } else { for (DatanodeStorageInfo storage : blocksMap.getStorages(b)) { checkReplicaOnStorage(numberReplicas, b, storage, nodesCorrupt, inStartupSafeMode); } } return numberReplicas; }
/** * Decrement number of safe blocks if the current block is contiguous * and it has just fallen below minimal replication or * if the current block is striped and its actual data blocks has just fallen * below the number of data units specified by erasure coding policy. * If safe mode is not currently on, this is a no-op. */ synchronized void decrementSafeBlockCount(BlockInfo b) { assert namesystem.hasWriteLock(); if (status == BMSafeModeStatus.OFF) { return; } final int safeNumberOfNodes = b.isStriped() ? ((BlockInfoStriped)b).getRealDataBlockNum() : safeReplication; BlockInfo storedBlock = blockManager.getStoredBlock(b); if (storedBlock.isComplete() && blockManager.countNodes(b).liveReplicas() == safeNumberOfNodes - 1) { this.blockSafe--; assert blockSafe >= 0; checkSafeMode(); } }
/** * Remove the block from the block map; * remove it from all data-node lists it belongs to; * and remove all data-node locations associated with the block. */ void removeBlock(BlockInfo block) { BlockInfo blockInfo = blocks.remove(block); if (blockInfo == null) { return; } decrementBlockStat(block); assert blockInfo.getBlockCollectionId() == INodeId.INVALID_INODE_ID; final int size = blockInfo.isStriped() ? blockInfo.getCapacity() : blockInfo.numNodes(); for(int idx = size - 1; idx >= 0; idx--) { DatanodeDescriptor dn = blockInfo.getDatanode(idx); if (dn != null) { removeBlock(dn, blockInfo); // remove from the list and wipe the location } } }
private void chooseExcessRedundancies( final Collection<DatanodeStorageInfo> nonExcess, BlockInfo storedBlock, short replication, DatanodeDescriptor addedNode, DatanodeDescriptor delNodeHint) { assert namesystem.hasWriteLock(); // first form a rack to datanodes map and BlockCollection bc = getBlockCollection(storedBlock); if (storedBlock.isStriped()) { chooseExcessRedundancyStriped(bc, nonExcess, storedBlock, delNodeHint); } else { final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy( bc.getStoragePolicyID()); final List<StorageType> excessTypes = storagePolicy.chooseExcess( replication, DatanodeStorageInfo.toStorageTypes(nonExcess)); chooseExcessRedundancyContiguous(nonExcess, storedBlock, replication, addedNode, delNodeHint, excessTypes); } }