/** @return the default policy. */ public BlockStoragePolicy getDefaultPolicy() { return getPolicy(defaultPolicyID); }
@Override public BlockStoragePolicy getStoragePolicy(byte policyId) { return createDefaultSuite.getPolicy(policyId); }
public BlockStoragePolicy getStoragePolicy(final byte policyId) { return storagePolicySuite.getPolicy(policyId); }
public BlockStoragePolicy getStoragePolicy(final String policyName) { return storagePolicySuite.getPolicy(policyName); }
/** Choose target for getting additional datanodes for an existing pipeline. */ public DatanodeStorageInfo[] chooseTarget4AdditionalDatanode(String src, int numAdditionalNodes, Node clientnode, List<DatanodeStorageInfo> chosen, Set<Node> excludes, long blocksize, byte storagePolicyID, BlockType blockType) { final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(storagePolicyID); final BlockPlacementPolicy blockplacement = placementPolicies.getPolicy(blockType); return blockplacement.chooseTarget(src, numAdditionalNodes, clientnode, chosen, true, excludes, blocksize, storagePolicy, null); }
final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy( bc.getStoragePolicyID()); final List<StorageType> excessTypes = storagePolicy.chooseExcess(
getDatanodeDescriptors(favoredNodes); final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(storagePolicyID); final BlockPlacementPolicy blockplacement = placementPolicies.getPolicy(blockType);
@Override void chooseTargets(BlockPlacementPolicy blockplacement, BlockStoragePolicySuite storagePolicySuite, Set<Node> excludedNodes) { // TODO: new placement policy for EC considering multiple writers DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget( getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0], getLiveReplicaStorages(), false, excludedNodes, getBlockSize(), storagePolicySuite.getPolicy(getStoragePolicyID()), null); setTargets(chosenTargets); }
private void chooseExcessRedundancies( final Collection<DatanodeStorageInfo> nonExcess, BlockInfo storedBlock, short replication, DatanodeDescriptor addedNode, DatanodeDescriptor delNodeHint) { assert namesystem.hasWriteLock(); // first form a rack to datanodes map and BlockCollection bc = getBlockCollection(storedBlock); if (storedBlock.isStriped()) { chooseExcessRedundancyStriped(bc, nonExcess, storedBlock, delNodeHint); } else { final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy( bc.getStoragePolicyID()); final List<StorageType> excessTypes = storagePolicy.chooseExcess( replication, DatanodeStorageInfo.toStorageTypes(nonExcess)); chooseExcessRedundancyContiguous(nonExcess, storedBlock, replication, addedNode, delNodeHint, excessTypes); } }
private static void verifyQuotaForTruncate(FSNamesystem fsn, INodesInPath iip, INodeFile file, long newLength, QuotaCounts delta) throws QuotaExceededException { FSDirectory fsd = fsn.getFSDirectory(); if (!fsn.isImageLoaded() || fsd.shouldSkipQuotaChecks()) { // Do not check quota if edit log is still being processed return; } final BlockStoragePolicy policy = fsd.getBlockStoragePolicySuite() .getPolicy(file.getStoragePolicyID()); file.computeQuotaDeltaForTruncate(newLength, policy, delta); fsd.readLock(); try { FSDirectory.verifyQuota(iip, iip.length() - 1, delta, null); } finally { fsd.readUnlock(); } }
@Override void chooseTargets(BlockPlacementPolicy blockplacement, BlockStoragePolicySuite storagePolicySuite, Set<Node> excludedNodes) { assert getSrcNodes().length > 0 : "At least 1 source node should have been selected"; try { DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget( getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0], getLiveReplicaStorages(), false, excludedNodes, getBlockSize(), storagePolicySuite.getPolicy(getStoragePolicyID()), null); setTargets(chosenTargets); } finally { getSrcNodes()[0].decrementPendingReplicationWithoutTargets(); } }
public void cleanFile(INode.ReclaimContext reclaimContext, final INodeFile file, final int snapshotId, int priorSnapshotId, byte storagePolicyId) { if (snapshotId == Snapshot.CURRENT_STATE_ID) { // delete the current file while the file has snapshot feature if (!isCurrentFileDeleted()) { file.recordModification(priorSnapshotId); deleteCurrentFile(); } final BlockStoragePolicy policy = reclaimContext.storagePolicySuite() .getPolicy(storagePolicyId); QuotaCounts old = file.storagespaceConsumed(policy); collectBlocksAndClear(reclaimContext, file); QuotaCounts current = file.storagespaceConsumed(policy); reclaimContext.quotaDelta().add(old.subtract(current)); } else { // delete the snapshot priorSnapshotId = getDiffs().updatePrior(snapshotId, priorSnapshotId); diffs.deleteSnapshotDiff(reclaimContext, snapshotId, priorSnapshotId, file); } }
/** Compute quota change for converting a complete block to a UC block. */ private static QuotaCounts computeQuotaDeltaForUCBlock(FSNamesystem fsn, INodeFile file) { final QuotaCounts delta = new QuotaCounts.Builder().build(); final BlockInfo lastBlock = file.getLastBlock(); if (lastBlock != null) { final long diff = file.getPreferredBlockSize() - lastBlock.getNumBytes(); final short repl = lastBlock.getReplication(); delta.addStorageSpace(diff * repl); final BlockStoragePolicy policy = fsn.getFSDirectory() .getBlockStoragePolicySuite().getPolicy(file.getStoragePolicyID()); List<StorageType> types = policy.chooseStorageTypes(repl); for (StorageType t : types) { if (t.supportTypeQuota()) { delta.addTypeSpace(t, diff); } } } return delta; } }
@Override public final ContentSummaryComputationContext computeContentSummary( int snapshotId, final ContentSummaryComputationContext summary) { final ContentCounts counts = summary.getCounts(); counts.addContent(Content.FILE, 1); final long fileLen = computeFileSize(snapshotId); counts.addContent(Content.LENGTH, fileLen); counts.addContent(Content.DISKSPACE, storagespaceConsumed(null) .getStorageSpace()); if (getStoragePolicyID() != BLOCK_STORAGE_POLICY_ID_UNSPECIFIED){ BlockStoragePolicy bsp = summary.getBlockStoragePolicySuite(). getPolicy(getStoragePolicyID()); List<StorageType> storageTypes = bsp.chooseStorageTypes(getFileReplication()); for (StorageType t : storageTypes) { if (!t.supportTypeQuota()) { continue; } counts.addTypeSpace(t, fileLen); } } return summary; }
/** Choose target for getting additional datanodes for an existing pipeline. */ public DatanodeStorageInfo[] chooseTarget4AdditionalDatanode(String src, int numAdditionalNodes, Node clientnode, List<DatanodeStorageInfo> chosen, Set<Node> excludes, long blocksize, byte storagePolicyID) { final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(storagePolicyID); return blockplacement.chooseTarget(src, numAdditionalNodes, clientnode, chosen, true, excludes, blocksize, storagePolicy); }
private static QuotaCounts computeQuotaDeltas(FSDirectory fsd, INodeFile target, INodeFile[] srcList) { QuotaCounts deltas = new QuotaCounts.Builder().build(); final short targetRepl = target.getPreferredBlockReplication(); for (INodeFile src : srcList) { short srcRepl = src.getFileReplication(); long fileSize = src.computeFileSize(); if (targetRepl != srcRepl) { deltas.addStorageSpace(fileSize * (targetRepl - srcRepl)); BlockStoragePolicy bsp = fsd.getBlockStoragePolicySuite().getPolicy(src.getStoragePolicyID()); if (bsp != null) { List<StorageType> srcTypeChosen = bsp.chooseStorageTypes(srcRepl); for (StorageType t : srcTypeChosen) { if (t.supportTypeQuota()) { deltas.addTypeSpace(t, -fileSize); } } List<StorageType> targetTypeChosen = bsp.chooseStorageTypes(targetRepl); for (StorageType t : targetTypeChosen) { if (t.supportTypeQuota()) { deltas.addTypeSpace(t, fileSize); } } } } } return deltas; }
private static boolean isLazyPersist(HdfsFileStatus stat) { final BlockStoragePolicy p = blockStoragePolicySuite.getPolicy( HdfsConstants.MEMORY_STORAGE_POLICY_NAME); return p != null && stat.getStoragePolicy() == p.getId(); }
bsps.getPolicy(blockStoragePolicyId); FileWithSnapshotFeature sf = getFileWithSnapshotFeature(); if (sf == null) {
if (storagePolicyID != HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) { bsp = reclaimContext.storagePolicySuite(). getPolicy(file.getStoragePolicyID());
private void chooseTargets(BlockPlacementPolicy blockplacement, BlockStoragePolicySuite storagePolicySuite, Set<Node> excludedNodes) { try { targets = blockplacement.chooseTarget(getSrcPath(), additionalReplRequired, srcNode, liveReplicaStorages, false, excludedNodes, block.getNumBytes(), storagePolicySuite.getPolicy(getStoragePolicyID())); } finally { srcNode.decrementPendingReplicationWithoutTargets(); } }