@InterfaceAudience.Private public static class NotEnoughReplicasException extends Exception { private static final long serialVersionUID = 1L; NotEnoughReplicasException(String msg) { super(msg); } }
/** Choose target for WebHDFS redirection. */ public DatanodeStorageInfo[] chooseTarget4WebHDFS(String src, DatanodeDescriptor clientnode, Set<Node> excludes, long blocksize) { return placementPolicies.getPolicy(CONTIGUOUS).chooseTarget(src, 1, clientnode, Collections.<DatanodeStorageInfo>emptyList(), false, excludes, blocksize, storagePolicySuite.getDefaultPolicy(), null); }
/** Choose target for getting additional datanodes for an existing pipeline. */ public DatanodeStorageInfo[] chooseTarget4AdditionalDatanode(String src, int numAdditionalNodes, Node clientnode, List<DatanodeStorageInfo> chosen, Set<Node> excludes, long blocksize, byte storagePolicyID, BlockType blockType) { final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(storagePolicyID); final BlockPlacementPolicy blockplacement = placementPolicies.getPolicy(blockType); return blockplacement.chooseTarget(src, numAdditionalNodes, clientnode, chosen, true, excludes, blocksize, storagePolicy, null); }
final BlockPlacementPolicy blockplacement = placementPolicies.getPolicy(blockType); final DatanodeStorageInfo[] targets = blockplacement.chooseTarget(src, numOfReplicas, client, excludedNodes, blocksize, favoredDatanodeDescriptors, storagePolicy, flags);
@Override void chooseTargets(BlockPlacementPolicy blockplacement, BlockStoragePolicySuite storagePolicySuite, Set<Node> excludedNodes) { // TODO: new placement policy for EC considering multiple writers DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget( getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0], getLiveReplicaStorages(), false, excludedNodes, getBlockSize(), storagePolicySuite.getPolicy(getStoragePolicyID()), null); setTargets(chosenTargets); }
@Override void chooseTargets(BlockPlacementPolicy blockplacement, BlockStoragePolicySuite storagePolicySuite, Set<Node> excludedNodes) { assert getSrcNodes().length > 0 : "At least 1 source node should have been selected"; try { DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget( getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0], getLiveReplicaStorages(), false, excludedNodes, getBlockSize(), storagePolicySuite.getPolicy(getStoragePolicyID()), null); setTargets(chosenTargets); } finally { getSrcNodes()[0].decrementPendingReplicationWithoutTargets(); } }
/** Choose target for WebHDFS redirection. */ public DatanodeStorageInfo[] chooseTarget4WebHDFS(String src, DatanodeDescriptor clientnode, Set<Node> excludes, long blocksize) { return blockplacement.chooseTarget(src, 1, clientnode, Collections.<DatanodeStorageInfo>emptyList(), false, excludes, blocksize, storagePolicySuite.getDefaultPolicy()); }
/** Choose target for WebHDFS redirection. */ public DatanodeStorageInfo[] chooseTarget4WebHDFS(String src, DatanodeDescriptor clientnode, Set<Node> excludes, long blocksize) { return blockplacement.chooseTarget(src, 1, clientnode, Collections.<DatanodeStorageInfo>emptyList(), false, excludes, blocksize, storagePolicySuite.getDefaultPolicy()); }
/** Choose target for getting additional datanodes for an existing pipeline. */ public DatanodeStorageInfo[] chooseTarget4AdditionalDatanode(String src, int numAdditionalNodes, Node clientnode, List<DatanodeStorageInfo> chosen, Set<Node> excludes, long blocksize, byte storagePolicyID) { final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(storagePolicyID); return blockplacement.chooseTarget(src, numAdditionalNodes, clientnode, chosen, true, excludes, blocksize, storagePolicy); }
@InterfaceAudience.Private public static class NotEnoughReplicasException extends Exception { private static final long serialVersionUID = 1L; NotEnoughReplicasException(String msg) { super(msg); } }
@InterfaceAudience.Private public static class NotEnoughReplicasException extends Exception { private static final long serialVersionUID = 1L; NotEnoughReplicasException(String msg) { super(msg); } }
/** Choose target for getting additional datanodes for an existing pipeline. */ public DatanodeStorageInfo[] chooseTarget4AdditionalDatanode(String src, int numAdditionalNodes, Node clientnode, List<DatanodeStorageInfo> chosen, Set<Node> excludes, long blocksize, byte storagePolicyID) { final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(storagePolicyID); return blockplacement.chooseTarget(src, numAdditionalNodes, clientnode, chosen, true, excludes, blocksize, storagePolicy); }
private static DatanodeStorageInfo[] chooseTarget( int numOfReplicas, DatanodeDescriptor writer, List<DatanodeStorageInfo> chosenNodes, Set<Node> excludedNodes) { return replicator.chooseTarget(filename, numOfReplicas, writer, chosenNodes, false, excludedNodes, BLOCK_SIZE, TestBlockStoragePolicy.DEFAULT_STORAGE_POLICY); }
private DatanodeStorageInfo[] chooseTarget( int numOfReplicas, DatanodeDescriptor writer, List<DatanodeStorageInfo> chosenNodes, Set<Node> excludedNodes) { return replicator.chooseTarget(filename, numOfReplicas, writer, chosenNodes, false, excludedNodes, BLOCK_SIZE, TestBlockStoragePolicy.DEFAULT_STORAGE_POLICY); }
getDatanodeDescriptors(favoredNodes); final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(storagePolicyID); final DatanodeStorageInfo[] targets = blockplacement.chooseTarget(src, numOfReplicas, client, excludedNodes, blocksize, favoredDatanodeDescriptors, storagePolicy);
getDatanodeDescriptors(favoredNodes); final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(storagePolicyID); final DatanodeStorageInfo[] targets = blockplacement.chooseTarget(src, numOfReplicas, client, excludedNodes, blocksize, favoredDatanodeDescriptors, storagePolicy);
private void chooseTargets(BlockPlacementPolicy blockplacement, BlockStoragePolicySuite storagePolicySuite, Set<Node> excludedNodes) { try { targets = blockplacement.chooseTarget(getSrcPath(), additionalReplRequired, srcNode, liveReplicaStorages, false, excludedNodes, block.getNumBytes(), storagePolicySuite.getPolicy(getStoragePolicyID())); } finally { srcNode.decrementPendingReplicationWithoutTargets(); } }
private void chooseTargets(BlockPlacementPolicy blockplacement, BlockStoragePolicySuite storagePolicySuite, Set<Node> excludedNodes) { try { targets = blockplacement.chooseTarget(bc.getName(), additionalReplRequired, srcNode, liveReplicaStorages, false, excludedNodes, block.getNumBytes(), storagePolicySuite.getPolicy(bc.getStoragePolicyID())); } finally { srcNode.decrementPendingReplicationWithoutTargets(); } } }
DatanodeStorageInfo[] targets = replicator.chooseTarget("/foo", 3, dataNodes[0], Collections.<DatanodeStorageInfo>emptyList(), false, new HashSet<Node>(), 0, policy);
@Test public void testChooseTarget() { int total = 0; int moreRemainingNode = 0; for (int i = 0; i < chooseTimes; i++) { DatanodeStorageInfo[] targets = namenode .getNamesystem() .getBlockManager() .getBlockPlacementPolicy() .chooseTarget(file, replica, null, new ArrayList<DatanodeStorageInfo>(), false, null, blockSize, TestBlockStoragePolicy.DEFAULT_STORAGE_POLICY); Assert.assertTrue(targets.length == replica); for (int j = 0; j < replica; j++) { total++; if (targets[j].getDatanodeDescriptor().getRemainingPercent() > 60) { moreRemainingNode++; } } } Assert.assertTrue(total == replica * chooseTimes); double possibility = 1.0 * moreRemainingNode / total; Assert.assertTrue(possibility > 0.52); Assert.assertTrue(possibility < 0.55); }