new ArrayList<DatanodeStorageInfo>(); NumberReplicas numReplicas = new NumberReplicas(); assert containingLiveReplicasNodes.size() >= numReplicas.liveReplicas(); int usableReplicas = numReplicas.liveReplicas() + numReplicas.decommissionedAndDecommissioning(); " live: " + numReplicas.liveReplicas() + " decommissioning and decommissioned: " + numReplicas.decommissionedAndDecommissioning() + " corrupt: " + numReplicas.corruptReplicas() + " in excess: " + numReplicas.excessReplicas() + " maintenance mode: " + numReplicas.maintenanceReplicas() + ") ");
out.println("No. of Expected Replica: " + blockManager.getExpectedRedundancyNum(blockInfo)); out.println("No. of live Replica: " + numberReplicas.liveReplicas()); out.println("No. of excess Replica: " + numberReplicas.excessReplicas()); out.println("No. of stale Replica: " + numberReplicas.replicasOnStaleNodes()); out.println("No. of decommissioned Replica: " + numberReplicas.decommissioned()); out.println("No. of decommissioning Replica: " + numberReplicas.decommissioning()); if (this.showMaintenanceState) { out.println("No. of entering maintenance Replica: " + numberReplicas.liveEnteringMaintenanceReplicas()); out.println("No. of in maintenance Replica: " + numberReplicas.maintenanceNotForReadReplicas()); numberReplicas.corruptReplicas());
NumberReplicas(int live, int decommissioned, int decommissioning, int corrupt, int excess, int stale) { initialize(live, decommissioned, decommissioning, corrupt, excess, stale); }
/** * * @return decommissioned and decommissioning replicas */ public int decommissionedAndDecommissioning() { return decommissioned() + decommissioning(); }
private boolean shouldProcessExtraRedundancy(NumberReplicas num, int expectedNum) { final int numCurrent = num.liveReplicas(); return numCurrent > expectedNum || (numCurrent == expectedNum && num.redundantInternalBlocks() > 0); }
public int outOfServiceReplicas() { return maintenanceReplicas() + decommissionedAndDecommissioning(); }
out.println("Block belongs to: "+iNode.getFullPathName()); out.println("No. of Expected Replica: " + bc.getBlockReplication()); out.println("No. of live Replica: " + numberReplicas.liveReplicas()); out.println("No. of excess Replica: " + numberReplicas.excessReplicas()); out.println("No. of stale Replica: " + numberReplicas.replicasOnStaleNodes()); out.println("No. of decommission Replica: " + numberReplicas.decommissioned()); out.println("No. of corrupted Replica: " + numberReplicas.corruptReplicas());
private static void logBlockReplicationInfo(Block block, BlockCollection bc, DatanodeDescriptor srcNode, NumberReplicas num, Iterable<DatanodeStorageInfo> storages) { int curReplicas = num.liveReplicas(); int curExpectedReplicas = bc.getBlockReplication(); StringBuilder nodeList = new StringBuilder(); for (DatanodeStorageInfo storage : storages) { final DatanodeDescriptor node = storage.getDatanodeDescriptor(); nodeList.append(node); nodeList.append(" "); } LOG.info("Block: " + block + ", Expected Replicas: " + curExpectedReplicas + ", live replicas: " + curReplicas + ", corrupt replicas: " + num.corruptReplicas() + ", decommissioned replicas: " + num.decommissioned() + ", decommissioning replicas: " + num.decommissioning() + ", excess replicas: " + num.excessReplicas() + ", Is Open File: " + bc.isUnderConstruction() + ", Datanodes having this block: " + nodeList + ", Current Datanode: " + srcNode + ", Is current datanode decommissioning: " + srcNode.isDecommissionInProgress()); }
/** * Check replication of the blocks in the collection. * If any block is needed replication, insert it into the replication queue. * Otherwise, if the block is more than the expected replication factor, * process it as an over replicated block. */ public void checkReplication(BlockCollection bc) { final short expected = bc.getBlockReplication(); for (Block block : bc.getBlocks()) { final NumberReplicas n = countNodes(block); if (isNeededReplication(block, expected, n.liveReplicas())) { neededReplications.add(block, n.liveReplicas(), n.decommissionedAndDecommissioning(), expected); } else if (n.liveReplicas() > expected) { processOverReplicatedBlock(block, expected, null, null); } } }
int curReplicas = num.liveReplicas(); int curExpectedRedundancy = blockManager.getExpectedRedundancyNum(block); StringBuilder nodeList = new StringBuilder(); "Block: " + block + ", Expected Replicas: " + curExpectedRedundancy + ", live replicas: " + curReplicas + ", corrupt replicas: " + num.corruptReplicas() + ", decommissioned replicas: " + num.decommissioned() + ", decommissioning replicas: " + num.decommissioning() + ", maintenance replicas: " + num.maintenanceReplicas() + ", live entering maintenance replicas: " + num.liveEnteringMaintenanceReplicas() + ", excess replicas: " + num.excessReplicas() + ", Is Open File: " + bc.isUnderConstruction() + ", Datanodes having this block: " + nodeList + ", Current Datanode: "
boolean hasEnoughLiveReplicas = numberOfReplicas.liveReplicas() >= expectedRedundancies; numberOfReplicas.liveReplicas()); (numberOfReplicas.liveReplicas() + numberOfReplicas.corruptReplicas()) > expectedRedundancies; boolean corruptedDuringWrite = minReplicationSatisfied &&
boolean hasEnoughEffectiveReplicas(BlockInfo block, NumberReplicas numReplicas, int pendingReplicaNum) { int required = getExpectedLiveRedundancyNum(block, numReplicas); int numEffectiveReplicas = numReplicas.liveReplicas() + pendingReplicaNum; return (numEffectiveReplicas >= required) && (pendingReplicaNum > 0 || isPlacementPolicySatisfied(block)); }
private void validateNumberReplicas(int expectedReplicas) throws IOException { NumberReplicas numberReplicas = blockManager.countNodes(block); assertThat(numberReplicas.liveReplicas(), is(expectedReplicas)); assertThat(numberReplicas.excessReplicas(), is(0)); assertThat(numberReplicas.corruptReplicas(), is(0)); assertThat(numberReplicas.decommissionedAndDecommissioning(), is(0)); assertThat(numberReplicas.replicasOnStaleNodes(), is(0)); BlockManagerTestUtil.updateState(blockManager); assertThat(blockManager.getUnderReplicatedBlocksCount(), is(0L)); assertThat(blockManager.getExcessBlocksCount(), is(0L)); }
int numCurrentReplica = num.liveReplicas(); .decommissionedAndDecommissioning(), expectedReplication)) { return MisReplicationResult.UNDER_REPLICATED; if (num.replicasOnStaleNodes() > 0) {
int numLiveReplicas = num.liveReplicas(); int numCurrentReplica = numLiveReplicas + pendingReplications.getNumReplicas(storedBlock); if (!isNeededReplication(storedBlock, fileReplication, numCurrentReplica)) { neededReplications.remove(storedBlock, numCurrentReplica, num.decommissionedAndDecommissioning(), fileReplication); } else { updateNeededReplications(storedBlock, curReplicaDelta, 0); int numCorruptNodes = num.corruptReplicas(); if (numCorruptNodes != corruptReplicasCount) { LOG.warn("Inconsistent number of corrupt replicas for " +
/** updates a block in needed reconstruction queue. */ private void updateNeededReconstructions(final BlockInfo block, final int curReplicasDelta, int expectedReplicasDelta) { namesystem.writeLock(); try { if (!isPopulatingReplQueues() || !block.isComplete()) { return; } NumberReplicas repl = countNodes(block); int pendingNum = pendingReconstruction.getNumReplicas(block); int curExpectedReplicas = getExpectedRedundancyNum(block); if (!hasEnoughEffectiveReplicas(block, repl, pendingNum)) { neededReconstruction.update(block, repl.liveReplicas() + pendingNum, repl.readOnlyReplicas(), repl.outOfServiceReplicas(), curExpectedReplicas, curReplicasDelta, expectedReplicasDelta); } else { int oldReplicas = repl.liveReplicas() + pendingNum - curReplicasDelta; int oldExpectedReplicas = curExpectedReplicas-expectedReplicasDelta; neededReconstruction.remove(block, oldReplicas, repl.readOnlyReplicas(), repl.outOfServiceReplicas(), oldExpectedReplicas); } } finally { namesystem.writeUnlock(); } }
NumberReplicas numReplicas = new NumberReplicas(); List<Byte> liveBlockIndices = new ArrayList<>(); final DatanodeDescriptor[] srcNodes = chooseSourceDatanodes(block, assert liveReplicaNodes.size() >= numReplicas.liveReplicas(); if (numReplicas.liveReplicas() < requiredRedundancy) { additionalReplRequired = requiredRedundancy - numReplicas.liveReplicas() - pendingNum; } else { if (additionalReplRequired - numReplicas.decommissioning() - numReplicas.liveEnteringMaintenanceReplicas() > 0) { additionalReplRequired = additionalReplRequired - numReplicas.decommissioning() - numReplicas.liveEnteringMaintenanceReplicas();
int decommissionedReplicas = numberReplicas.decommissioned(); int decommissioningReplicas = numberReplicas.decommissioning(); int enteringMaintenanceReplicas = numberReplicas.liveEnteringMaintenanceReplicas(); int inMaintenanceReplicas = numberReplicas.maintenanceNotForReadReplicas(); res.decommissionedReplicas += decommissionedReplicas; res.decommissioningReplicas += decommissioningReplicas; int liveReplicas = numberReplicas.liveReplicas(); int totalReplicasPerBlock = liveReplicas + decommissionedReplicas + decommissioningReplicas
void checkTimeout(String testLabel, long cycleTime) throws TimeoutException { if (Time.monotonicNow() > failtime) { throw new TimeoutException("Timeout: " + testLabel + " for block " + lastBlock + " after " + timeout + " msec. Last counts: live = " + lastNum.liveReplicas() + ", excess = " + lastNum.excessReplicas() + ", corrupt = " + lastNum.corruptReplicas()); } if (cycleTime > 0) { try { Thread.sleep(cycleTime); } catch (InterruptedException ie) { //ignore } } }
if (nr.replicasOnStaleNodes() > 0) { blockLog.info("BLOCK* invalidateBlocks: postponing " + "invalidation of {} on {} because {} replica(s) are located on " + "nodes with potentially out-of-date block reports", b, dn, nr.replicasOnStaleNodes()); postponeBlock(b.corrupted); return false; } else if (nr.liveReplicas() >= 1) {