datanodeManager.getNetworkTopology(), datanodeManager.getHost2DatanodeMap()); storagePolicySuite = BlockStoragePolicySuite.createDefaultSuite(); pendingReconstruction = new PendingReconstructionBlocks(conf.getInt( DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY,
public BlockStoragePolicy[] getStoragePolicies() { return storagePolicySuite.getAllPolicies(); }
/** @return the corresponding policy. */ public BlockStoragePolicy getPolicy(byte id) { // id == 0 means policy not specified. return id == 0? getDefaultPolicy(): policies[id]; }
/** @return the default policy. */ public BlockStoragePolicy getDefaultPolicy() { return getPolicy(defaultPolicyID); }
@Test public void testMultipleHots() { BlockStoragePolicySuite bsps = BlockStoragePolicySuite.createDefaultSuite(); StoragePolicySummary sts = new StoragePolicySummary(bsps.getAllPolicies()); BlockStoragePolicy hot = bsps.getPolicy("HOT"); sts.add(new StorageType[]{StorageType.DISK},hot); sts.add(new StorageType[]{StorageType.DISK,StorageType.DISK},hot); sts.add(new StorageType[]{StorageType.DISK, StorageType.DISK,StorageType.DISK},hot); sts.add(new StorageType[]{StorageType.DISK, StorageType.DISK,StorageType.DISK,StorageType.DISK},hot); Map<String, Long> actualOutput = convertToStringMap(sts); Assert.assertEquals(4,actualOutput.size()); Map<String, Long> expectedOutput = new HashMap<>(); expectedOutput.put("HOT|DISK:1(HOT)", 1l); expectedOutput.put("HOT|DISK:2(HOT)", 1l); expectedOutput.put("HOT|DISK:3(HOT)", 1l); expectedOutput.put("HOT|DISK:4(HOT)", 1l); Assert.assertEquals(expectedOutput,actualOutput); }
.createDefaultSuite(); final BlockStoragePolicy warm = suite.getPolicy("WARM"); final BlockStoragePolicy cold = suite.getPolicy("COLD"); DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /foo", 0, "The storage policy of " + foo.toString() + ":\n" + warm);
final BlockStoragePolicy policy = POLICY_SUITE.getPolicy(i); if (policy != null) { final String s = policy.toString(); Assert.assertEquals(POLICY_SUITE.getPolicy(HOT), POLICY_SUITE.getDefaultPolicy()); final BlockStoragePolicy cold = POLICY_SUITE.getPolicy(COLD); for(short replication = 1; replication < 6; replication++) { final List<StorageType> computed = cold.chooseStorageTypes(replication); final BlockStoragePolicy warm = POLICY_SUITE.getPolicy(WARM); for(short replication = 1; replication < 6; replication++) { final List<StorageType> computed = warm.chooseStorageTypes(replication); final BlockStoragePolicy hot = POLICY_SUITE.getPolicy(HOT); for(short replication = 1; replication < 6; replication++) { final List<StorageType> computed = hot.chooseStorageTypes(replication);
List<DatanodeStorageInfo> excessReplicas = new ArrayList<>(); BlockStoragePolicySuite POLICY_SUITE = BlockStoragePolicySuite .createDefaultSuite(); BlockStoragePolicy storagePolicy = POLICY_SUITE.getDefaultPolicy(); DatanodeStorageInfo excessSSD = DFSTestUtil.createDatanodeStorageInfo( "Storage-excess-SSD-ID", "localhost",
public static XAttr buildXAttr(byte policyId) { final String name = buildXAttrName(); return XAttrHelper.buildXAttr(name, new byte[]{policyId}); }
private static void setDirStoragePolicy( FSDirectory fsd, INodesInPath iip, byte policyId) throws IOException { INode inode = FSDirectory.resolveLastINode(iip); List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode); XAttr xAttr = BlockStoragePolicySuite.buildXAttr(policyId); List<XAttr> newXAttrs = null; if (policyId == HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) { List<XAttr> toRemove = Lists.newArrayList(); toRemove.add(xAttr); List<XAttr> removed = Lists.newArrayList(); newXAttrs = FSDirXAttrOp.filterINodeXAttrs(existingXAttrs, toRemove, removed); } else { newXAttrs = FSDirXAttrOp.setINodeXAttrs(fsd, existingXAttrs, Arrays.asList(xAttr), EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); } XAttrStorage.updateINodeXAttrs(inode, newXAttrs, iip.getLatestSnapshotId()); }
new StorageType[]{StorageType.PROVIDED, StorageType.DISK}, new StorageType[]{StorageType.PROVIDED, StorageType.DISK}); return new BlockStoragePolicySuite(hotId, policies);
@Override public byte getLocalStoragePolicyID() { XAttrFeature f = getXAttrFeature(); ImmutableList<XAttr> xattrs = f == null ? ImmutableList.<XAttr> of() : f .getXAttrs(); for (XAttr xattr : xattrs) { if (BlockStoragePolicySuite.isStoragePolicyXAttr(xattr)) { return (xattr.getValue())[0]; } } return ID_UNSPECIFIED; }
@Override public BlockStoragePolicy getStoragePolicy(byte policyId) { return createDefaultSuite.getPolicy(policyId); }
@Test public void testMultipleHotsWithDifferentCounts() { BlockStoragePolicySuite bsps = BlockStoragePolicySuite.createDefaultSuite(); StoragePolicySummary sts = new StoragePolicySummary(bsps.getAllPolicies()); BlockStoragePolicy hot = bsps.getPolicy("HOT"); sts.add(new StorageType[]{StorageType.DISK},hot); sts.add(new StorageType[]{StorageType.DISK,StorageType.DISK},hot); sts.add(new StorageType[]{StorageType.DISK,StorageType.DISK},hot); sts.add(new StorageType[]{StorageType.DISK, StorageType.DISK,StorageType.DISK},hot); sts.add(new StorageType[]{StorageType.DISK, StorageType.DISK,StorageType.DISK},hot); sts.add(new StorageType[]{StorageType.DISK, StorageType.DISK,StorageType.DISK,StorageType.DISK},hot); Map<String, Long> actualOutput = convertToStringMap(sts); Assert.assertEquals(4,actualOutput.size()); Map<String, Long> expectedOutput = new HashMap<>(); expectedOutput.put("HOT|DISK:1(HOT)", 1l); expectedOutput.put("HOT|DISK:2(HOT)", 2l); expectedOutput.put("HOT|DISK:3(HOT)", 2l); expectedOutput.put("HOT|DISK:4(HOT)", 1l); Assert.assertEquals(expectedOutput,actualOutput); }
public static XAttr buildXAttr(byte policyId) { final String name = buildXAttrName(); return XAttrHelper.buildXAttr(name, new byte[]{policyId}); }
private static void setDirStoragePolicy( FSDirectory fsd, INodeDirectory inode, byte policyId, int latestSnapshotId) throws IOException { List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode); XAttr xAttr = BlockStoragePolicySuite.buildXAttr(policyId); List<XAttr> newXAttrs = FSDirXAttrOp.setINodeXAttrs(fsd, existingXAttrs, Arrays.asList(xAttr), EnumSet.of( XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); XAttrStorage.updateINodeXAttrs(inode, newXAttrs, latestSnapshotId); }
new StorageType[]{StorageType.ARCHIVE}, StorageType.EMPTY_ARRAY, StorageType.EMPTY_ARRAY); return new BlockStoragePolicySuite(hotId, policies);
@Override public byte getLocalStoragePolicyID() { XAttrFeature f = getXAttrFeature(); ImmutableList<XAttr> xattrs = f == null ? ImmutableList.<XAttr> of() : f .getXAttrs(); for (XAttr xattr : xattrs) { if (BlockStoragePolicySuite.isStoragePolicyXAttr(xattr)) { return (xattr.getValue())[0]; } } return ID_UNSPECIFIED; }
public BlockStoragePolicy getStoragePolicy(final byte policyId) { return storagePolicySuite.getPolicy(policyId); }
@Test public void testSortInDescendingOrder() { BlockStoragePolicySuite bsps = BlockStoragePolicySuite.createDefaultSuite(); StoragePolicySummary sts = new StoragePolicySummary(bsps.getAllPolicies()); BlockStoragePolicy hot = bsps.getPolicy("HOT"); BlockStoragePolicy warm = bsps.getPolicy("WARM"); BlockStoragePolicy cold = bsps.getPolicy("COLD");