public void removeBlockFromMap(BlockInfo block) { for(DatanodeStorageInfo info : blocksMap.getStorages(block)) { excessRedundancyMap.remove(info.getDatanodeDescriptor(), block); } blocksMap.removeBlock(block); // If block is removed from blocksMap remove it from corruptReplicasMap corruptReplicas.removeFromCorruptReplicasMap(block); }
public void clear() { blockIdManager.clear(); clearQueues(); blocksMap.clear(); }
public static LocatedBlock newLocatedBlock(ExtendedBlock eb, BlockInfo info, DatanodeStorageInfo[] locs, long offset) throws IOException { final LocatedBlock lb; if (info.isStriped()) { lb = newLocatedStripedBlock(eb, locs, info.getUnderConstructionFeature().getBlockIndices(), offset, false); } else { lb = newLocatedBlock(eb, locs, offset, false); } return lb; }
@Override void chooseTargets(BlockPlacementPolicy blockplacement, BlockStoragePolicySuite storagePolicySuite, Set<Node> excludedNodes) { // TODO: new placement policy for EC considering multiple writers DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget( getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0], getLiveReplicaStorages(), false, excludedNodes, getBlockSize(), storagePolicySuite.getPolicy(getStoragePolicyID()), null); setTargets(chosenTargets); }
void addStoredBlockUnderConstruction(StatefulBlockInfo ucBlock, DatanodeStorageInfo storageInfo) throws IOException { BlockInfo block = ucBlock.storedBlock; block.getUnderConstructionFeature().addReplicaIfNotPresent( storageInfo, ucBlock.reportedBlock, ucBlock.reportedState); // Add replica if appropriate. If the replica was previously corrupt // but now okay, it might need to be updated. if (ucBlock.reportedState == ReplicaState.FINALIZED && ( block.findStorageInfo(storageInfo) < 0) || corruptReplicas .isReplicaCorrupt(block, storageInfo.getDatanodeDescriptor())) { addStoredBlock(block, ucBlock.reportedBlock, storageInfo, null, true); } }
/** Increment the number of blocks scheduled for each given storage */ public static void incrementBlocksScheduled(DatanodeStorageInfo... storages) { for (DatanodeStorageInfo s : storages) { s.getDatanodeDescriptor().incrementBlocksScheduled(s.getStorageType()); } }
boolean isNeededReconstructionForMaintenance(BlockInfo storedBlock, NumberReplicas numberReplicas) { return storedBlock.isComplete() && (numberReplicas.liveReplicas() < getMinMaintenanceStorageNum(storedBlock) || !isPlacementPolicySatisfied(storedBlock)); }
boolean hasEnoughEffectiveReplicas(BlockInfo block, NumberReplicas numReplicas, int pendingReplicaNum) { int required = getExpectedLiveRedundancyNum(block, numReplicas); int numEffectiveReplicas = numReplicas.liveReplicas() + pendingReplicaNum; return (numEffectiveReplicas >= required) && (pendingReplicaNum > 0 || isPlacementPolicySatisfied(block)); }
public boolean hasStorageType(StorageType type) { for (DatanodeStorageInfo dnStorage : getStorageInfos()) { if (dnStorage.getStorageType() == type) { return true; } } return false; } }
/** * Force the given block in the given file to be marked as complete, * regardless of whether enough replicas are present. This is necessary * when tailing edit logs as a Standby. */ public void forceCompleteBlock(final BlockInfo block) throws IOException { List<ReplicaUnderConstruction> staleReplicas = block.commitBlock(block); removeStaleReplicas(staleReplicas, block); completeBlock(block, null, true); }
@Override boolean removeBlock(BlockInfo b) { ProvidedDescriptor dn = (ProvidedDescriptor) getDatanodeDescriptor(); if (dn.activeProvidedDatanodes() == 0) { return super.removeBlock(b); } else { return false; } }
public DatanodeStorageInfo[] getStorages(BlockInfo block) { final DatanodeStorageInfo[] storages = new DatanodeStorageInfo[block.numNodes()]; int i = 0; for(DatanodeStorageInfo s : blocksMap.getStorages(block)) { storages[i++] = s; } return storages; }
/** * Is data-node the replica belongs to alive. */ boolean isAlive() { return expectedLocation.getDatanodeDescriptor().isAlive(); }
public BlockIdManager(BlockManager blockManager) { this.legacyGenerationStampLimit = HdfsConstants.GRANDFATHER_GENERATION_STAMP; this.blockIdGenerator = new SequentialBlockIdGenerator(blockManager); this.blockGroupIdGenerator = new SequentialBlockGroupIdGenerator(blockManager); }
/** * Adds block to list of blocks which will be invalidated on specified * datanode and log the operation */ void addToInvalidates(final Block block, final DatanodeInfo datanode) { if (!isPopulatingReplQueues()) { return; } invalidateBlocks.add(block, datanode, true); }
public short getExpectedRedundancyNum(BlockInfo block) { return block.isStriped() ? ((BlockInfoStriped) block).getRealTotalBlockNum() : block.getReplication(); }
@Override public BlockPlacementStatus verifyBlockPlacement(DatanodeInfo[] locs, int numberOfReplicas) { BlockPlacementStatus defaultStatus = super.verifyBlockPlacement(locs, numberOfReplicas); BlockPlacementStatusWithUpgradeDomain upgradeDomainStatus = new BlockPlacementStatusWithUpgradeDomain(defaultStatus, getUpgradeDomainsFromNodes(locs), numberOfReplicas, upgradeDomainFactor); return upgradeDomainStatus; }
private Block getBlockOnStorage(BlockInfo storedBlock, DatanodeStorageInfo storage) { return storedBlock.isStriped() ? ((BlockInfoStriped) storedBlock).getBlockOnStorage(storage) : storedBlock; }
/** * Create array of expected replica locations * (as has been assigned by chooseTargets()). */ public DatanodeStorageInfo[] getExpectedStorageLocations() { int numLocations = getNumExpectedLocations(); DatanodeStorageInfo[] storages = new DatanodeStorageInfo[numLocations]; for (int i = 0; i < numLocations; i++) { storages[i] = replicas[i].getExpectedStorageLocation(); } return storages; }