this.blockManager = new BlockManager(this, haEnabled, conf); this.datanodeStatistics = blockManager.getDatanodeManager().getDatanodeStatistics(); CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, ""), blockManager.getStoragePolicySuite().getDefaultPolicy().getId());
switch (rdbi.getStatus()) { case DELETED_BLOCK: removeStoredBlock(storageInfo, rdbi.getBlock(), node); deleted++; break; case RECEIVED_BLOCK: addBlock(storageInfo, rdbi.getBlock(), rdbi.getDelHints()); received++; break; case RECEIVING_BLOCK: receiving++; processAndHandleReportedBlock(storageInfo, rdbi.getBlock(), ReplicaState.RBW, null); break;
isGenStampInFuture(block)) { queueReportedBlock(storageInfo, block, reportedState, QUEUE_REASON_FUTURE_GENSTAMP); return; BlockInfo storedBlock = getStoredBlock(block); if(storedBlock == null) { addToInvalidates(new Block(block), node); return; BlockToMarkCorrupt c = checkReplicaCorrupt( block, reportedState, storedBlock, ucState, node); if (c != null) { queueReportedBlock(storageInfo, storedBlock, reportedState, QUEUE_REASON_CORRUPT_STATE); } else { markBlockAsCorrupt(c, storageInfo, node); if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) { addStoredBlockUnderConstruction( new StatefulBlockInfo(storedBlock, new Block(block), reportedState), storageInfo); && (storedBlock.findStorageInfo(storageInfo) == -1 || corruptReplicas.isReplicaCorrupt(storedBlock, node))) { addStoredBlock(storedBlock, block, storageInfo, delHintNode, true);
/** Set replication for the blocks. */ public void setReplication( final short oldRepl, final short newRepl, final BlockInfo b) { if (newRepl == oldRepl) { return; } // update neededReconstruction priority queues b.setReplication(newRepl); NumberReplicas num = countNodes(b); updateNeededReconstructions(b, 0, newRepl - oldRepl); if (shouldProcessExtraRedundancy(num, newRepl)) { processExtraRedundancyBlock(b, newRepl, null, null); } }
public boolean hasMinStorage(BlockInfo block) { return countNodes(block).liveReplicas() >= getMinStorageNum(block); }
/** * Check if a block is replicated to at least the minimum replication. */ public boolean isSufficientlyReplicated(BlockInfo b) { // Compare against the lesser of the minReplication and number of live DNs. final int replication = Math.min(minReplication, getDatanodeManager().getNumLiveDataNodes()); return countNodes(b).liveReplicas() >= replication; }
NumberReplicas nr) throws IOException { blockLog.debug("BLOCK* invalidateBlock: {} on {}", b, dn); DatanodeDescriptor node = getDatanodeManager().getDatanode(dn); if (node == null) { throw new IOException("Cannot invalidate " + b "nodes with potentially out-of-date block reports", b, dn, nr.replicasOnStaleNodes()); postponeBlock(b.getCorrupted()); return false; } else { addToInvalidates(b.getCorrupted(), dn); removeStoredBlock(b.getStored(), node); blockLog.debug("BLOCK* invalidateBlocks: {} on {} listed for deletion.", b, dn);
BlockInfoContiguous blockInfo = bm.getStoredBlock(block); if(blockInfo == null) { out.println("Block "+ blockId +" " + NONEXISTENT_STATUS); return; BlockCollection bc = bm.getBlockCollection(blockInfo); INode iNode = (INode) bc; NumberReplicas numberReplicas= bm.countNodes(block); out.println("Block Id: " + blockId); out.println("Block belongs to: "+iNode.getFullPathName()); if (bm.getCorruptReplicas(block) != null) { corruptionRecord = bm.getCorruptReplicas(block); if (corruptionRecord != null && corruptionRecord.contains(dn)) { out.print(CORRUPT_STATUS+"\t ReasonCode: "+ bm.getCorruptReason(block,dn)); } else if (dn.isDecommissioned() ){ out.print(DECOMMISSIONED_STATUS);
return null; assert lastBlock == getStoredBlock(lastBlock) : "last block of the file is not in blocksMap"; DatanodeStorageInfo[] targets = getStorages(lastBlock); NumberReplicas replicas = countNodes(lastBlock); neededReconstruction.remove(lastBlock, replicas.liveReplicas(), replicas.readOnlyReplicas(), replicas.outOfServiceReplicas(), getExpectedRedundancyNum(lastBlock)); pendingReconstruction.remove(lastBlock); final Block b = getBlockOnStorage(lastBlock, storage); if (b != null) { invalidateBlocks.remove(storage.getDatanodeDescriptor(), b); hasMinStorage(lastBlock, targets.length) ? -1 : 0, getStoragePolicySuite()).getLength(); final long pos = fileLength - lastBlock.getNumBytes(); return createLocatedBlock(null, lastBlock, pos, BlockTokenIdentifier.AccessMode.WRITE);
storedBlock = getStoredBlock(block); } else { storedBlock = block; NumberReplicas num = countNodes(storedBlock); int numLiveReplicas = num.liveReplicas(); int pendingNum = pendingReconstruction.getNumReplicas(storedBlock); hasMinStorage(storedBlock, numLiveReplicas)) { addExpectedReplicasToPending(storedBlock); completeBlock(storedBlock, null, false); } else if (storedBlock.isComplete() && result == AddBlockResult.ADDED) { if (!isPopulatingReplQueues()) { return storedBlock; short fileRedundancy = getExpectedRedundancyNum(storedBlock); if (!isNeededReconstruction(storedBlock, num, pendingNum)) { neededReconstruction.remove(storedBlock, numCurrentReplica, num.readOnlyReplicas(), num.outOfServiceReplicas(), fileRedundancy); } else { updateNeededReconstructions(storedBlock, curReplicaDelta, 0); if (shouldProcessExtraRedundancy(num, fileRedundancy)) { processExtraRedundancyBlock(storedBlock, fileRedundancy, node, delNodeHint); invalidateCorruptReplicas(storedBlock, reportedBlock, num);
final DatanodeDescriptor node = storageInfo.getDatanodeDescriptor(); processReportedBlock(storageInfo, block, reportedState, toAdd, toInvalidate, toCorrupt, toUC); addStoredBlockUnderConstruction(b, storageInfo); addStoredBlock(b, storageInfo, delHintNode, numBlocksLogged < maxNumBlocksToLog); numBlocksLogged++; blockLog.info("BLOCK* addBlock: block {} on node {} size {} does not " + "belong to any file", b, node, b.getNumBytes()); addToInvalidates(b, node); markBlockAsCorrupt(b, storageInfo, node);
if (blockManager.getMissingBlocksCount() == 0) { if (cookieTab[0] == null) { cookieTab[0] = String.valueOf(getIntCookie(cookieTab[0])); final Iterator<Block> blkIterator = blockManager.getCorruptReplicaBlockIterator(); final INode inode = (INode)blockManager.getBlockCollection(blk); skip++; if (inode != null && blockManager.countNodes(blk).liveReplicas() == 0) { String src = inode.getFullPathName(); if (src.startsWith(path)){
reportDiffSorted(storageInfo, sortedReport, toAdd, toRemove, toInvalidate, toCorrupt, toUC); addStoredBlockUnderConstruction(b, storageInfo); removeStoredBlock(b, node); addStoredBlock(b.stored, b.reported, storageInfo, null, numBlocksLogged < maxNumBlocksToLog); numBlocksLogged++; addToInvalidates(b, node); markBlockAsCorrupt(b, storageInfo, node);
if (bc == null) { addToInvalidates(block); return MisReplicationResult.INVALID; NumberReplicas num = countNodes(block); int numCurrentReplica = num.liveReplicas(); if (isNeededReplication(block, expectedReplication, numCurrentReplica)) { if (neededReplications.add(block, numCurrentReplica, num .decommissionedAndDecommissioning(), expectedReplication)) { processOverReplicatedBlock(block, expectedReplication, null, null); return MisReplicationResult.OVER_REPLICATED;
Collection<BlockToMarkCorrupt> toCorrupt = new LinkedList<BlockToMarkCorrupt>(); Collection<StatefulBlockInfo> toUC = new LinkedList<StatefulBlockInfo>(); reportDiff(storageInfo, report, toAdd, toRemove, toInvalidate, toCorrupt, toUC); addStoredBlockUnderConstruction(b, storageInfo); removeStoredBlock(b, node); addStoredBlock(b, storageInfo, null, numBlocksLogged < maxNumBlocksToLog); numBlocksLogged++; addToInvalidates(b, node); markBlockAsCorrupt(b, storageInfo, node);
BlockInfo blockInfo = blockManager.getStoredBlock(block); if (blockInfo == null || blockInfo.isDeleted()) { out.println("Block "+ blockId +" " + NONEXISTENT_STATUS); NumberReplicas numberReplicas= blockManager.countNodes(blockInfo); out.println("Block Id: " + blockId); out.println("Block belongs to: "+iNode.getFullPathName()); out.println("No. of Expected Replica: " + blockManager.getExpectedRedundancyNum(blockInfo)); out.println("No. of live Replica: " + numberReplicas.liveReplicas()); out.println("No. of excess Replica: " + numberReplicas.excessReplicas()); if (blockManager.getCorruptReplicas(block) != null) { corruptionRecord = blockManager.getCorruptReplicas(block); if (corruptionRecord != null && corruptionRecord.contains(dn)) { out.print(CORRUPT_STATUS + "\t ReasonCode: " + blockManager.getCorruptReason(block, dn)); } else if (dn.isDecommissioned() ){ out.print(DECOMMISSIONED_STATUS);
blockLog.debug("BLOCK markBlockAsCorrupt: {} cannot be marked as" + " corrupt as it does not belong to any file", b); addToInvalidates(b.getCorrupted(), node); return; getExpectedRedundancyNum(b.getStored()); b.getReasonCode(), b.getStored().isStriped()); NumberReplicas numberOfReplicas = countNodes(b.getStored()); boolean hasEnoughLiveReplicas = numberOfReplicas.liveReplicas() >= expectedRedundancies; boolean minReplicationSatisfied = hasMinStorage(b.getStored(), numberOfReplicas.liveReplicas()); || corruptedDuringWrite) { invalidateBlock(b, node, numberOfReplicas); } else if (isPopulatingReplQueues()) { updateNeededReconstructions(b.getStored(), -1, 0);
/** * Reformatted DataNodes will replace the original UUID in the * {@link DatanodeManager#datanodeMap}. This tests if block * invalidation work on the original DataNode can be skipped. */ @Test(timeout=120000) public void testDatanodeReformat() throws Exception { namesystem.writeLock(); try { // Change the datanode UUID to emulate a reformat String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnr = cluster.getDataNode(nodes[0].getIpcPort()) .getDNRegistrationForBP(poolId); dnr = new DatanodeRegistration(UUID.randomUUID().toString(), dnr); cluster.stopDataNode(nodes[0].getXferAddr()); Block block = new Block(0, 0, GenerationStamp.LAST_RESERVED_STAMP); bm.addToInvalidates(block, nodes[0]); bm.getDatanodeManager().registerDatanode(dnr); // Since UUID has changed, the invalidation work should be skipped assertEquals(0, bm.computeInvalidateWork(1)); assertEquals(0, bm.getPendingDeletionBlocksCount()); } finally { namesystem.writeUnlock(); } }
* this.blocksInvalidateWorkPct); int workFound = this.computeReplicationWork(blocksToProcess); this.updateState(); this.scheduledReplicationBlocksCount = workFound; } finally { namesystem.writeUnlock(); workFound += this.computeInvalidateWork(nodesToProcess); return workFound;
final boolean committed = commitBlock(lastBlock, commitBlock); if (committed && lastBlock.isStriped()) { NumberReplicas numReplicas = countNodes(lastBlock); int numUsableReplicas = numReplicas.liveReplicas() + numReplicas.decommissioning() + numReplicas.liveEnteringMaintenanceReplicas(); if (hasMinStorage(lastBlock, numUsableReplicas)) { if (committed) { addExpectedReplicasToPending(lastBlock); completeBlock(lastBlock, iip, false); } else if (pendingRecoveryBlocks.isUnderRecovery(lastBlock)) { completeBlock(lastBlock, iip, true); updateNeededReconstructions(lastBlock, 1, 0);