BlockToMarkCorrupt(BlockInfoContiguous stored, long gs, String reason, Reason reasonCode) { this(new BlockInfoContiguous(stored), stored, reason, reasonCode); //the corrupted block in datanode has a different generation stamp corrupted.setGenerationStamp(gs); }
@Override boolean removeStorage(DatanodeStorageInfo storage) { int dnIndex = findStorageInfo(storage); if (dnIndex < 0) { // the node is not found return false; } // find the last not null node int lastNode = numNodes()-1; // replace current node entry by the lastNode one setStorageInfo(dnIndex, getStorageInfo(lastNode)); // set the last entry to null setStorageInfo(lastNode, null); return true; }
@Override boolean addStorage(DatanodeStorageInfo storage, Block reportedBlock) { Preconditions.checkArgument(this.getBlockId() == reportedBlock.getBlockId(), "reported blk_%s is different from stored blk_%s", reportedBlock.getBlockId(), this.getBlockId()); // find the last null node int lastNode = ensureCapacity(1); setStorageInfo(lastNode, storage); return true; }
@Override public int numNodes() { assert this.storages != null : "BlockInfo is not initialized"; for (int idx = getCapacity()-1; idx >= 0; idx--) { if (getDatanode(idx) != null) { return idx + 1; } } return 0; }
/** * Remove {@link DatanodeStorageInfo} location for a block */ boolean removeStorage(DatanodeStorageInfo storage) { int dnIndex = findStorageInfo(storage); if(dnIndex < 0) // the node is not found return false; assert getPrevious(dnIndex) == null && getNext(dnIndex) == null : "Block is still in the list and must be removed first."; // find the last not null node int lastNode = numNodes()-1; // replace current node triplet by the lastNode one setStorageInfo(dnIndex, getStorageInfo(lastNode)); setNext(dnIndex, getNext(lastNode)); setPrevious(dnIndex, getPrevious(lastNode)); // set the last triplet to null setStorageInfo(lastNode, null); setNext(lastNode, null); setPrevious(lastNode, null); return true; }
final long oldGenerationStamp = storedBlock.getGenerationStamp(); final long oldNumBytes = storedBlock.getNumBytes(); BlockCollection blockCollection = storedBlock.getBlockCollection(); if (blockCollection == null) { throw new IOException("The blockCollection of " + storedBlock + src + ", likely due to delayed block removal"); if ((!iFile.isUnderConstruction() || storedBlock.isComplete()) && iFile.getLastBlock().isComplete()) { if (LOG.isDebugEnabled()) { LOG.debug("Unexpected block (=" + oldBlock long recoveryId = truncatedBlock.getBlockRecoveryId(); boolean copyTruncate = truncatedBlock.getBlockId() != storedBlock.getBlockId(); if(recoveryId != newgenerationstamp) { throw new IOException("The recovery id " + newgenerationstamp storedBlock.setGenerationStamp(newgenerationstamp); storedBlock.setNumBytes(newlength);
/** * Insert this block into the head of the list of blocks * related to the specified DatanodeStorageInfo. * If the head is null then form a new list. * @return current block as the new head of the list. */ BlockInfoContiguous listInsert(BlockInfoContiguous head, DatanodeStorageInfo storage) { int dnIndex = this.findStorageInfo(storage); assert dnIndex >= 0 : "Data node is not found: current"; assert getPrevious(dnIndex) == null && getNext(dnIndex) == null : "Block is already in the list and cannot be inserted."; this.setPrevious(dnIndex, null); this.setNext(dnIndex, head); if(head != null) head.setPrevious(head.findStorageInfo(storage), this); return this; }
if (oldBlock.getBlockId() != newBlock.getBlockId() || (oldBlock.getGenerationStamp() != newBlock.getGenerationStamp() && !(isGenStampUpdate && isLastBlock))) { throw new IOException("Mismatched block IDs or generation stamps, " + oldBlock.setNumBytes(newBlock.getNumBytes()); boolean changeMade = oldBlock.getGenerationStamp() != newBlock.getGenerationStamp(); oldBlock.setGenerationStamp(newBlock.getGenerationStamp()); newBI = new BlockInfoContiguous(newBlock, file.getBlockReplication());
if (!blockInfo.isComplete()) { LOG.debug("Block {}: can't cache this block, because it is not yet" + " complete.", cachedBlock.getBlockId()); int numReplicas = blockInfo.getCapacity(); Collection<DatanodeDescriptor> corrupt = blockManager.getCorruptReplicas(blockInfo); int outOfCapacity = 0; for (int i = 0; i < numReplicas; i++) { DatanodeDescriptor datanode = blockInfo.getDatanode(i); if (datanode == null) { continue; blockManager.getStoredBlock(new Block(cBlock.getBlockId())); if (info != null) { pendingBytes -= info.getNumBytes(); blockManager.getStoredBlock(new Block(cBlock.getBlockId())); if (info != null) { pendingBytes += info.getNumBytes(); if (pendingCapacity < blockInfo.getNumBytes()) { LOG.trace("Block {}: DataNode {} is not a valid possibility " + "because the block has size {}, but the DataNode only has {}" + "bytes of cache remaining ({} pending bytes, {} already cached.", blockInfo.getBlockId(), datanode.getDatanodeUuid(), blockInfo.getNumBytes(), pendingCapacity, pendingBytes, datanode.getCacheRemaining()); outOfCapacity++;
/** * Replace a block in the block map by a new block. * The new block and the old one have the same key. * @param newBlock - block for replacement * @return new block */ BlockInfoContiguous replaceBlock(BlockInfoContiguous newBlock) { BlockInfoContiguous currentBlock = blocks.get(newBlock); assert currentBlock != null : "the block if not in blocksMap"; // replace block in data-node lists for (int i = currentBlock.numNodes() - 1; i >= 0; i--) { final DatanodeDescriptor dn = currentBlock.getDatanode(i); final DatanodeStorageInfo storage = currentBlock.findStorageInfo(dn); final boolean removed = storage.removeBlock(currentBlock); Preconditions.checkState(removed, "currentBlock not found."); final AddBlockResult result = storage.addBlock(newBlock); Preconditions.checkState(result == AddBlockResult.ADDED, "newBlock already exists."); } // replace block in the map itself blocks.put(newBlock); return newBlock; } }
/** * Remove the block from the block map; * remove it from all data-node lists it belongs to; * and remove all data-node locations associated with the block. */ void removeBlock(Block block) { BlockInfoContiguous blockInfo = blocks.remove(block); if (blockInfo == null) return; blockInfo.setBlockCollection(null); for(int idx = blockInfo.numNodes()-1; idx >= 0; idx--) { DatanodeDescriptor dn = blockInfo.getDatanode(idx); dn.removeBlock(blockInfo); // remove from the list and wipe the location } }
/** * Add a {@link DatanodeStorageInfo} location for a block */ boolean addStorage(DatanodeStorageInfo storage) { // find the last null node int lastNode = ensureCapacity(1); setStorageInfo(lastNode, storage); setNext(lastNode, null); setPrevious(lastNode, null); return true; }
if(newBlock == null) { newBlock = (shouldCopyOnTruncate) ? createNewBlock() : new Block(oldBlock.getBlockId(), oldBlock.getNumBytes(), nextGenerationStamp(blockIdManager.isLegacyBlock(oldBlock))); truncatedBlockUC.setNumBytes(oldBlock.getNumBytes() - lastBlockDelta); truncatedBlockUC.setTruncateBlock(oldBlock); file.setLastBlock(truncatedBlockUC, blockManager.getStorages(oldBlock)); assert !oldBlock.isComplete() : "oldBlock should be under construction"; truncatedBlockUC = (BlockInfoContiguousUnderConstruction) oldBlock; truncatedBlockUC.setTruncateBlock(new Block(oldBlock)); truncatedBlockUC.getTruncateBlock().setNumBytes( oldBlock.getNumBytes() - lastBlockDelta); truncatedBlockUC.getTruncateBlock().setGenerationStamp( newBlock.getGenerationStamp());
if (!blockInfo.getBlockUCState().equals(BlockUCState.COMPLETE)) { blockInfo.getBlockUCState() ); continue; Block block = new Block(blockInfo.getBlockId()); CachedBlock ncblock = new CachedBlock(block.getBlockId(), directive.getReplication(), mark); ocblock.getDatanodes(Type.CACHED); long cachedByBlock = Math.min(cachedOn.size(), directive.getReplication()) * blockInfo.getNumBytes(); cachedTotal += cachedByBlock;
/** * Find specified DatanodeStorageInfo. * @return index or -1 if not found. */ int findStorageInfo(DatanodeStorageInfo storageInfo) { int len = getCapacity(); for(int idx = 0; idx < len; idx++) { DatanodeStorageInfo cur = getStorageInfo(idx); if (cur == storageInfo) { return idx; } if (cur == null) { break; } } return -1; }
@Override public Boolean get() { BlockInfoContiguous info = blockManager.getStoredBlock(b.getLocalBlock()); int count = 0; StringBuilder sb = new StringBuilder("Replica locations: "); for (int i = 0; i < info.numNodes(); i++) { DatanodeDescriptor dn = info.getDatanode(i); sb.append(dn + ", "); if (!dn.getDatanodeUuid().equals(uuid)) { count++; } } LOG.info(sb.toString()); LOG.info("Count: " + count); return count == replicas; } }, 500, 30000);
blocks[j] = new BlockInfoContiguous(replication); blocks[j].readFields(in);
public AddBlockResult addBlock(BlockInfoContiguous b) { // First check whether the block belongs to a different storage // on the same DN. AddBlockResult result = AddBlockResult.ADDED; DatanodeStorageInfo otherStorage = b.findStorageInfo(getDatanodeDescriptor()); if (otherStorage != null) { if (otherStorage != this) { // The block belongs to a different storage. Remove it first. otherStorage.removeBlock(b); result = AddBlockResult.REPLACED; } else { // The block is already associated with this storage. return AddBlockResult.ALREADY_EXIST; } } // add to the head of the data-node list b.addStorage(this); blockList = b.listInsert(blockList, this); numBlocks++; return result; }
if (storedBlock.getBlockUCState() == BlockUCState.COMMITTED && numCurrentReplica >= minReplication) { completeBlock(storedBlock.getBlockCollection(), storedBlock, null, false); } else if (storedBlock.isComplete() && result == AddBlockResult.ADDED) {
/** * Convert a complete block to an under construction block. * @return BlockInfoUnderConstruction - an under construction block. */ public BlockInfoContiguousUnderConstruction convertToBlockUnderConstruction( BlockUCState s, DatanodeStorageInfo[] targets) { if(isComplete()) { BlockInfoContiguousUnderConstruction ucBlock = new BlockInfoContiguousUnderConstruction(this, getBlockCollection().getBlockReplication(), s, targets); ucBlock.setBlockCollection(getBlockCollection()); return ucBlock; } // the block is already under construction BlockInfoContiguousUnderConstruction ucBlock = (BlockInfoContiguousUnderConstruction)this; ucBlock.setBlockUCState(s); ucBlock.setExpectedLocations(targets); ucBlock.setBlockCollection(getBlockCollection()); return ucBlock; }