@Override boolean useDelHint(DatanodeStorageInfo delHint, DatanodeStorageInfo added, List<DatanodeStorageInfo> moreThanOne, Collection<DatanodeStorageInfo> exactlyOne, List<StorageType> excessTypes) { if (!super.useDelHint(delHint, added, moreThanOne, exactlyOne, excessTypes)) { // If BlockPlacementPolicyDefault doesn't allow useDelHint, there is no // point checking with upgrade domain policy. return false; } return isMovableBasedOnUpgradeDomain(combine(moreThanOne, exactlyOne), delHint, added); }
private <T> Map<String, List<T>> getUpgradeDomainMap( Collection<T> storagesOrDataNodes) { Map<String, List<T>> upgradeDomainMap = new HashMap<>(); for(T storage : storagesOrDataNodes) { String upgradeDomain = getUpgradeDomainWithDefaultValue( getDatanodeInfo(storage)); List<T> storages = upgradeDomainMap.get(upgradeDomain); if (storages == null) { storages = new ArrayList<>(); upgradeDomainMap.put(upgradeDomain, storages); } storages.add(storage); } return upgradeDomainMap; }
private <T> boolean isMovableBasedOnUpgradeDomain(Collection<T> all, T source, T target) { Map<String, List<T>> udMap = getUpgradeDomainMap(all); // shareUDSet includes datanodes that share same upgrade // domain with another datanode. List<T> shareUDSet = getShareUDSet(udMap); // check if removing source reduces the number of upgrade domains if (notReduceNumOfGroups(shareUDSet, source, target)) { return true; } else if (udMap.size() > upgradeDomainFactor) { return true; // existing number of upgrade domain exceeds the limit. } else { return false; // removing source reduces the number of UDs. } }
@Override protected Collection<DatanodeStorageInfo> pickupReplicaSet( Collection<DatanodeStorageInfo> moreThanOne, Collection<DatanodeStorageInfo> exactlyOne, Map<String, List<DatanodeStorageInfo>> rackMap) { // shareUDSet includes DatanodeStorageInfo that share same upgrade // domain with another DatanodeStorageInfo. Collection<DatanodeStorageInfo> all = combine(moreThanOne, exactlyOne); List<DatanodeStorageInfo> shareUDSet = getShareUDSet( getUpgradeDomainMap(all)); // shareRackAndUDSet contains those DatanodeStorageInfo that // share rack and upgrade domain with another DatanodeStorageInfo. List<DatanodeStorageInfo> shareRackAndUDSet = new ArrayList<>(); if (shareUDSet.size() == 0) { // All upgrade domains are unique, use the parent set. return super.pickupReplicaSet(moreThanOne, exactlyOne, rackMap); } else if (moreThanOne != null) { for (DatanodeStorageInfo storage : shareUDSet) { if (moreThanOne.contains(storage)) { shareRackAndUDSet.add(storage); } } } return (shareRackAndUDSet.size() > 0) ? shareRackAndUDSet : shareUDSet; }
private Set<String> getUpgradeDomainsFromNodes(DatanodeInfo[] nodes) { Set<String> upgradeDomains = new HashSet<>(); if (nodes == null) { return upgradeDomains; } for(DatanodeInfo node : nodes) { upgradeDomains.add(getUpgradeDomainWithDefaultValue(node)); } return upgradeDomains; }
private Set<String> getUpgradeDomains(List<DatanodeStorageInfo> results) { Set<String> upgradeDomains = new HashSet<>(); if (results == null) { return upgradeDomains; } for(DatanodeStorageInfo storageInfo : results) { upgradeDomains.add(getUpgradeDomain(storageInfo)); } return upgradeDomains; }
@Override public boolean isMovable(Collection<DatanodeInfo> locs, DatanodeInfo source, DatanodeInfo target) { if (super.isMovable(locs, source, target)) { return isMovableBasedOnUpgradeDomain(locs, source, target); } else { return false; } } }
@Override protected boolean isGoodDatanode(DatanodeDescriptor node, int maxTargetPerRack, boolean considerLoad, List<DatanodeStorageInfo> results, boolean avoidStaleNodes) { boolean isGoodTarget = super.isGoodDatanode(node, maxTargetPerRack, considerLoad, results, avoidStaleNodes); if (isGoodTarget) { if (results.size() > 0 && results.size() < upgradeDomainFactor) { // Each node in "results" has a different upgrade domain. Make sure // the candidate node introduces a new upgrade domain. Set<String> upgradeDomains = getUpgradeDomains(results); if (upgradeDomains.contains(node.getUpgradeDomain())) { isGoodTarget = false; } } } return isGoodTarget; }
@Override public BlockPlacementStatus verifyBlockPlacement(DatanodeInfo[] locs, int numberOfReplicas) { BlockPlacementStatus defaultStatus = super.verifyBlockPlacement(locs, numberOfReplicas); BlockPlacementStatusWithUpgradeDomain upgradeDomainStatus = new BlockPlacementStatusWithUpgradeDomain(defaultStatus, getUpgradeDomainsFromNodes(locs), numberOfReplicas, upgradeDomainFactor); return upgradeDomainStatus; }
private String getUpgradeDomain(DatanodeStorageInfo storage) { return getUpgradeDomainWithDefaultValue(storage.getDatanodeDescriptor()); }