/** * Copy block references into the snapshot * up to the current {@link #fileSize}. * Should be done only once. */ public void setBlocks(BlockInfoContiguous[] blocks) { if(this.blocks != null) return; int numBlocks = 0; for(long s = 0; numBlocks < blocks.length && s < fileSize; numBlocks++) s += blocks[numBlocks].getNumBytes(); this.blocks = Arrays.copyOf(blocks, numBlocks); }
/** * Copy block references into the snapshot * up to the current {@link #fileSize}. * Should be done only once. */ public void setBlocks(BlockInfoContiguous[] blocks) { if(this.blocks != null) return; int numBlocks = 0; for(long s = 0; numBlocks < blocks.length && s < fileSize; numBlocks++) s += blocks[numBlocks].getNumBytes(); this.blocks = Arrays.copyOf(blocks, numBlocks); }
private LocatedBlock createLocatedBlock(final BlockInfoContiguous[] blocks, final long endPos, final AccessMode mode) throws IOException { int curBlk = 0; long curPos = 0; int nrBlocks = (blocks[0].getNumBytes() == 0) ? 0 : blocks.length; for (curBlk = 0; curBlk < nrBlocks; curBlk++) { long blkSize = blocks[curBlk].getNumBytes(); if (curPos + blkSize >= endPos) { break; } curPos += blkSize; } return createLocatedBlock(blocks[curBlk], curPos, mode); }
private LocatedBlock createLocatedBlock(final BlockInfoContiguous[] blocks, final long endPos, final AccessMode mode) throws IOException { int curBlk = 0; long curPos = 0; int nrBlocks = (blocks[0].getNumBytes() == 0) ? 0 : blocks.length; for (curBlk = 0; curBlk < nrBlocks; curBlk++) { long blkSize = blocks[curBlk].getNumBytes(); if (curPos + blkSize >= endPos) { break; } curPos += blkSize; } return createLocatedBlock(blocks[curBlk], curPos, mode); }
long size = blocks[last].getNumBytes(); if (blocks[last] instanceof BlockInfoContiguousUnderConstruction) { if (!includesLastUcBlock) { size += blocks[i].getNumBytes();
private void logAddStoredBlock(BlockInfoContiguous storedBlock, DatanodeDescriptor node) { if (!blockLog.isInfoEnabled()) { return; } StringBuilder sb = new StringBuilder(500); sb.append("BLOCK* addStoredBlock: blockMap updated: ") .append(node) .append(" is added to "); storedBlock.appendStringTo(sb); sb.append(" size " ) .append(storedBlock.getNumBytes()); blockLog.info(sb.toString()); } /**
private List<LocatedBlock> createLocatedBlockList( final BlockInfoContiguous[] blocks, final long offset, final long length, final int nrBlocksToReturn, final AccessMode mode) throws IOException { int curBlk = 0; long curPos = 0, blkSize = 0; int nrBlocks = (blocks[0].getNumBytes() == 0) ? 0 : blocks.length; for (curBlk = 0; curBlk < nrBlocks; curBlk++) { blkSize = blocks[curBlk].getNumBytes(); assert blkSize > 0 : "Block of size 0"; if (curPos + blkSize > offset) { break; } curPos += blkSize; } if (nrBlocks > 0 && curBlk == nrBlocks) // offset >= end of file return Collections.<LocatedBlock>emptyList(); long endOff = offset + length; List<LocatedBlock> results = new ArrayList<LocatedBlock>(blocks.length); do { results.add(createLocatedBlock(blocks[curBlk], curPos, mode)); curPos += blocks[curBlk].getNumBytes(); curBlk++; } while (curPos < endOff && curBlk < blocks.length && results.size() < nrBlocksToReturn); return results; }
private List<LocatedBlock> createLocatedBlockList( final BlockInfoContiguous[] blocks, final long offset, final long length, final int nrBlocksToReturn, final AccessMode mode) throws IOException { int curBlk = 0; long curPos = 0, blkSize = 0; int nrBlocks = (blocks[0].getNumBytes() == 0) ? 0 : blocks.length; for (curBlk = 0; curBlk < nrBlocks; curBlk++) { blkSize = blocks[curBlk].getNumBytes(); assert blkSize > 0 : "Block of size 0"; if (curPos + blkSize > offset) { break; } curPos += blkSize; } if (nrBlocks > 0 && curBlk == nrBlocks) // offset >= end of file return Collections.<LocatedBlock>emptyList(); long endOff = offset + length; List<LocatedBlock> results = new ArrayList<LocatedBlock>(blocks.length); do { results.add(createLocatedBlock(blocks[curBlk], curPos, mode)); curPos += blocks[curBlk].getNumBytes(); curBlk++; } while (curPos < endOff && curBlk < blocks.length && results.size() < nrBlocksToReturn); return results; }
private void logAddStoredBlock(BlockInfoContiguous storedBlock, DatanodeDescriptor node) { if (!blockLog.isInfoEnabled()) { return; } StringBuilder sb = new StringBuilder(500); sb.append("BLOCK* addStoredBlock: blockMap updated: ") .append(node) .append(" is added to "); storedBlock.appendStringTo(sb); sb.append(" size " ) .append(storedBlock.getNumBytes()); blockLog.info(sb.toString()); } /**
/** * Remove full blocks at the end file up to newLength * @return sum of sizes of the remained blocks */ public long collectBlocksBeyondMax(final long max, final BlocksMapUpdateInfo collectedBlocks) { final BlockInfoContiguous[] oldBlocks = getBlocks(); if (oldBlocks == null) return 0; // find the minimum n such that the size of the first n blocks > max int n = 0; long size = 0; for(; n < oldBlocks.length && max > size; n++) { size += oldBlocks[n].getNumBytes(); } if (n >= oldBlocks.length) return size; // starting from block n, the data is beyond max. // resize the array. truncateBlocksTo(n); // collect the blocks beyond max if (collectedBlocks != null) { for(; n < oldBlocks.length; n++) { collectedBlocks.addDeleteBlock(oldBlocks[n]); } } return size; }
/** * Remove full blocks at the end file up to newLength * @return sum of sizes of the remained blocks */ public long collectBlocksBeyondMax(final long max, final BlocksMapUpdateInfo collectedBlocks) { final BlockInfoContiguous[] oldBlocks = getBlocks(); if (oldBlocks == null) return 0; // find the minimum n such that the size of the first n blocks > max int n = 0; long size = 0; for(; n < oldBlocks.length && max > size; n++) { size += oldBlocks[n].getNumBytes(); } if (n >= oldBlocks.length) return size; // starting from block n, the data is beyond max. // resize the array. truncateBlocksTo(n); // collect the blocks beyond max if (collectedBlocks != null) { for(; n < oldBlocks.length; n++) { collectedBlocks.addDeleteBlock(oldBlocks[n]); } } return size; }
public final long storagespaceConsumedNoReplication() { FileWithSnapshotFeature sf = getFileWithSnapshotFeature(); if(sf == null) { return computeFileSize(true, true); } // Collect all distinct blocks long size = 0; Set<Block> allBlocks = new HashSet<Block>(Arrays.asList(getBlocks())); List<FileDiff> diffs = sf.getDiffs().asList(); for(FileDiff diff : diffs) { BlockInfoContiguous[] diffBlocks = diff.getBlocks(); if (diffBlocks != null) { allBlocks.addAll(Arrays.asList(diffBlocks)); } } for(Block block : allBlocks) { size += block.getNumBytes(); } // check if the last block is under construction BlockInfoContiguous lastBlock = getLastBlock(); if(lastBlock != null && lastBlock instanceof BlockInfoContiguousUnderConstruction) { size += getPreferredBlockSize() - lastBlock.getNumBytes(); } return size; }
public final long storagespaceConsumedNoReplication() { FileWithSnapshotFeature sf = getFileWithSnapshotFeature(); if(sf == null) { return computeFileSize(true, true); } // Collect all distinct blocks long size = 0; Set<Block> allBlocks = new HashSet<Block>(Arrays.asList(getBlocks())); List<FileDiff> diffs = sf.getDiffs().asList(); for(FileDiff diff : diffs) { BlockInfoContiguous[] diffBlocks = diff.getBlocks(); if (diffBlocks != null) { allBlocks.addAll(Arrays.asList(diffBlocks)); } } for(Block block : allBlocks) { size += block.getNumBytes(); } // check if the last block is under construction BlockInfoContiguous lastBlock = getLastBlock(); if(lastBlock != null && lastBlock instanceof BlockInfoContiguousUnderConstruction) { size += getPreferredBlockSize() - lastBlock.getNumBytes(); } return size; }
/** * Update the cached quota space for a block that is being completed. * Must only be called once, as the block is being completed. * @param completeBlk - Completed block for which to update space * @param inodes - INodes in path to file containing completeBlk; if null * this will be resolved internally */ public void updateSpaceForCompleteBlock(BlockInfoContiguous completeBlk, INodesInPath inodes) throws IOException { assert namesystem.hasWriteLock(); INodesInPath iip = inodes != null ? inodes : INodesInPath.fromINode((INodeFile) completeBlk.getBlockCollection()); INodeFile fileINode = iip.getLastINode().asFile(); // Adjust disk space consumption if required final long diff = fileINode.getPreferredBlockSize() - completeBlk.getNumBytes(); if (diff > 0) { try { updateSpaceConsumed(iip, 0, -diff, fileINode.getFileReplication()); } catch (IOException e) { LOG.warn("Unexpected exception while updating disk space.", e); } } }
/** * Update the cached quota space for a block that is being completed. * Must only be called once, as the block is being completed. * @param completeBlk - Completed block for which to update space * @param inodes - INodes in path to file containing completeBlk; if null * this will be resolved internally */ public void updateSpaceForCompleteBlock(BlockInfoContiguous completeBlk, INodesInPath inodes) throws IOException { assert namesystem.hasWriteLock(); INodesInPath iip = inodes != null ? inodes : INodesInPath.fromINode((INodeFile) completeBlk.getBlockCollection()); INodeFile fileINode = iip.getLastINode().asFile(); // Adjust disk space consumption if required final long diff = fileINode.getPreferredBlockSize() - completeBlk.getNumBytes(); if (diff > 0) { try { updateSpaceConsumed(iip, 0, -diff, fileINode.getFileReplication()); } catch (IOException e) { LOG.warn("Unexpected exception while updating disk space.", e); } } }
/** Compute quota change for converting a complete block to a UC block */ private QuotaCounts computeQuotaDeltaForUCBlock(INodeFile file) { final QuotaCounts delta = new QuotaCounts.Builder().build(); final BlockInfoContiguous lastBlock = file.getLastBlock(); if (lastBlock != null) { final long diff = file.getPreferredBlockSize() - lastBlock.getNumBytes(); final short repl = file.getBlockReplication(); delta.addStorageSpace(diff * repl); final BlockStoragePolicy policy = dir.getBlockStoragePolicySuite() .getPolicy(file.getStoragePolicyID()); List<StorageType> types = policy.chooseStorageTypes(repl); for (StorageType t : types) { if (t.supportTypeQuota()) { delta.addTypeSpace(t, diff); } } } return delta; }
/** Compute quota change for converting a complete block to a UC block */ private QuotaCounts computeQuotaDeltaForUCBlock(INodeFile file) { final QuotaCounts delta = new QuotaCounts.Builder().build(); final BlockInfoContiguous lastBlock = file.getLastBlock(); if (lastBlock != null) { final long diff = file.getPreferredBlockSize() - lastBlock.getNumBytes(); final short repl = file.getBlockReplication(); delta.addStorageSpace(diff * repl); final BlockStoragePolicy policy = dir.getBlockStoragePolicySuite() .getPolicy(file.getStoragePolicyID()); List<StorageType> types = policy.chooseStorageTypes(repl); for (StorageType t : types) { if (t.supportTypeQuota()) { delta.addTypeSpace(t, diff); } } } return delta; }
BlockInfoContiguous[] blks = barNode.getBlocks(); assertEquals(1, blks.length); assertEquals(BLOCKSIZE, blks[0].getNumBytes()); ExtendedBlock previous = new ExtendedBlock(fsn.getBlockPoolId(), blks[0]); cluster.getNameNodeRpc() blks = barNode.getBlocks(); assertEquals(2, blks.length); assertEquals(BLOCKSIZE, blks[0].getNumBytes()); assertEquals(0, blks[1].getNumBytes()); blks = barNode.getBlocks(); assertEquals(1, blks.length); assertEquals(BLOCKSIZE, blks[0].getNumBytes());
blks = barNode.getBlocks(); assertEquals(2, blks.length); assertEquals(BLOCKSIZE, blks[0].getNumBytes()); assertEquals(0, blks[1].getNumBytes()); blks = barNode.getBlocks(); assertEquals(1, blks.length); assertEquals(BLOCKSIZE, blks[0].getNumBytes());
BlockInfoContiguous[] blks = barNode.getBlocks(); assertEquals(1, blks.length); assertEquals(testData.length, blks[0].getNumBytes());