/** * Copy construction. * This is used to convert BlockInfoUnderConstruction * @param from BlockInfo to copy from. */ protected BlockInfoContiguous(BlockInfoContiguous from) { this(from, from.bc.getBlockReplication()); this.bc = from.bc; }
/** * Copy construction. * This is used to convert BlockInfoUnderConstruction * @param from BlockInfo to copy from. */ protected BlockInfoContiguous(BlockInfoContiguous from) { this(from, from.bc.getBlockReplication()); this.bc = from.bc; }
/** * @return 0 if the block is not found; * otherwise, return the replication factor of the block. */ private int getReplication(Block block) { final BlockCollection bc = blocksMap.getBlockCollection(block); return bc == null? 0: bc.getBlockReplication(); }
/** * @return 0 if the block is not found; * otherwise, return the replication factor of the block. */ private int getReplication(Block block) { final BlockCollection bc = blocksMap.getBlockCollection(block); return bc == null? 0: bc.getBlockReplication(); }
private BlockInfoContiguous addBlockOnNodes(long blockId, List<DatanodeDescriptor> nodes) { BlockCollection bc = Mockito.mock(BlockCollection.class); Mockito.doReturn((short)3).when(bc).getBlockReplication(); BlockInfoContiguous blockInfo = blockOnNodes(blockId, nodes); bm.blocksMap.addBlockCollection(blockInfo, bc); return blockInfo; }
BlockCollection bc, NumberReplicas numberReplicas) { final int numExpected = bc.getBlockReplication(); final int numLive = numberReplicas.liveReplicas(); if (numLive >= numExpected
private BlockInfoContiguous addBlockToBM(long blkId) { Block block = new Block(blkId); BlockInfoContiguous blockInfo = new BlockInfoContiguous(block, (short) 3); BlockCollection bc = Mockito.mock(BlockCollection.class); Mockito.doReturn((short) 3).when(bc).getBlockReplication(); bm.blocksMap.addBlockCollection(blockInfo, bc); return blockInfo; }
/** * On stopping decommission, check if the node has excess replicas. * If there are any excess replicas, call processOverReplicatedBlock(). * Process over replicated blocks only when active NN is out of safe mode. */ void processOverReplicatedBlocksOnReCommission( final DatanodeDescriptor srcNode) { if (!namesystem.isPopulatingReplQueues()) { return; } final Iterator<? extends Block> it = srcNode.getBlockIterator(); int numOverReplicated = 0; while(it.hasNext()) { final Block block = it.next(); BlockCollection bc = blocksMap.getBlockCollection(block); short expectedReplication = bc.getBlockReplication(); NumberReplicas num = countNodes(block); int numCurrentReplica = num.liveReplicas(); if (numCurrentReplica > expectedReplication) { // over-replicated block processOverReplicatedBlock(block, expectedReplication, null, null); numOverReplicated++; } } LOG.info("Invalidated " + numOverReplicated + " over-replicated blocks on " + srcNode + " during recommissioning"); }
private BlockInfoContiguous addUcBlockToBM(long blkId) { Block block = new Block(blkId); BlockInfoContiguousUnderConstruction blockInfo = new BlockInfoContiguousUnderConstruction(block, (short) 3); BlockCollection bc = Mockito.mock(BlockCollection.class); Mockito.doReturn((short) 3).when(bc).getBlockReplication(); bm.blocksMap.addBlockCollection(blockInfo, bc); return blockInfo; }
private static void logBlockReplicationInfo(Block block, BlockCollection bc, DatanodeDescriptor srcNode, NumberReplicas num, Iterable<DatanodeStorageInfo> storages) { int curReplicas = num.liveReplicas(); int curExpectedReplicas = bc.getBlockReplication(); StringBuilder nodeList = new StringBuilder(); for (DatanodeStorageInfo storage : storages) { final DatanodeDescriptor node = storage.getDatanodeDescriptor(); nodeList.append(node); nodeList.append(" "); } LOG.info("Block: " + block + ", Expected Replicas: " + curExpectedReplicas + ", live replicas: " + curReplicas + ", corrupt replicas: " + num.corruptReplicas() + ", decommissioned replicas: " + num.decommissioned() + ", decommissioning replicas: " + num.decommissioning() + ", excess replicas: " + num.excessReplicas() + ", Is Open File: " + bc.isUnderConstruction() + ", Datanodes having this block: " + nodeList + ", Current Datanode: " + srcNode + ", Is current datanode decommissioning: " + srcNode.isDecommissionInProgress()); }
private static void logBlockReplicationInfo(Block block, BlockCollection bc, DatanodeDescriptor srcNode, NumberReplicas num, Iterable<DatanodeStorageInfo> storages) { int curReplicas = num.liveReplicas(); int curExpectedReplicas = bc.getBlockReplication(); StringBuilder nodeList = new StringBuilder(); for (DatanodeStorageInfo storage : storages) { final DatanodeDescriptor node = storage.getDatanodeDescriptor(); nodeList.append(node); nodeList.append(" "); } LOG.info("Block: " + block + ", Expected Replicas: " + curExpectedReplicas + ", live replicas: " + curReplicas + ", corrupt replicas: " + num.corruptReplicas() + ", decommissioned replicas: " + num.decommissioned() + ", decommissioning replicas: " + num.decommissioning() + ", excess replicas: " + num.excessReplicas() + ", Is Open File: " + bc.isUnderConstruction() + ", Datanodes having this block: " + nodeList + ", Current Datanode: " + srcNode + ", Is current datanode decommissioning: " + srcNode.isDecommissionInProgress()); }
/** * On stopping decommission, check if the node has excess replicas. * If there are any excess replicas, call processOverReplicatedBlock(). * Process over replicated blocks only when active NN is out of safe mode. */ void processOverReplicatedBlocksOnReCommission( final DatanodeDescriptor srcNode) { if (!namesystem.isPopulatingReplQueues()) { return; } final Iterator<? extends Block> it = srcNode.getBlockIterator(); int numOverReplicated = 0; while(it.hasNext()) { final Block block = it.next(); BlockCollection bc = blocksMap.getBlockCollection(block); short expectedReplication = bc.getBlockReplication(); NumberReplicas num = countNodes(block); int numCurrentReplica = num.liveReplicas(); if (numCurrentReplica > expectedReplication) { // over-replicated block processOverReplicatedBlock(block, expectedReplication, null, null); numOverReplicated++; } } LOG.info("Invalidated " + numOverReplicated + " over-replicated blocks on " + srcNode + " during recommissioning"); }
.getBlockReplication(); boolean minReplicationSatisfied = numberOfReplicas.liveReplicas() >= minReplication; boolean hasMoreCorruptReplicas = minReplicationSatisfied && (numberOfReplicas.liveReplicas() + numberOfReplicas.corruptReplicas()) > bc.getBlockReplication(); boolean corruptedDuringWrite = minReplicationSatisfied && (b.stored.getGenerationStamp() > b.corrupted.getGenerationStamp());
/** * Check replication of the blocks in the collection. * If any block is needed replication, insert it into the replication queue. * Otherwise, if the block is more than the expected replication factor, * process it as an over replicated block. */ public void checkReplication(BlockCollection bc) { final short expected = bc.getBlockReplication(); for (Block block : bc.getBlocks()) { final NumberReplicas n = countNodes(block); if (isNeededReplication(block, expected, n.liveReplicas())) { neededReplications.add(block, n.liveReplicas(), n.decommissionedAndDecommissioning(), expected); } else if (n.liveReplicas() > expected) { processOverReplicatedBlock(block, expected, null, null); } } }
/** * Check replication of the blocks in the collection. * If any block is needed replication, insert it into the replication queue. * Otherwise, if the block is more than the expected replication factor, * process it as an over replicated block. */ public void checkReplication(BlockCollection bc) { final short expected = bc.getBlockReplication(); for (Block block : bc.getBlocks()) { final NumberReplicas n = countNodes(block); if (isNeededReplication(block, expected, n.liveReplicas())) { neededReplications.add(block, n.liveReplicas(), n.decommissionedAndDecommissioning(), expected); } else if (n.liveReplicas() > expected) { processOverReplicatedBlock(block, expected, null, null); } } }
/** * Convert a complete block to an under construction block. * @return BlockInfoUnderConstruction - an under construction block. */ public BlockInfoContiguousUnderConstruction convertToBlockUnderConstruction( BlockUCState s, DatanodeStorageInfo[] targets) { if(isComplete()) { BlockInfoContiguousUnderConstruction ucBlock = new BlockInfoContiguousUnderConstruction(this, getBlockCollection().getBlockReplication(), s, targets); ucBlock.setBlockCollection(getBlockCollection()); return ucBlock; } // the block is already under construction BlockInfoContiguousUnderConstruction ucBlock = (BlockInfoContiguousUnderConstruction)this; ucBlock.setBlockUCState(s); ucBlock.setExpectedLocations(targets); ucBlock.setBlockCollection(getBlockCollection()); return ucBlock; }
/** * Convert a complete block to an under construction block. * @return BlockInfoUnderConstruction - an under construction block. */ public BlockInfoContiguousUnderConstruction convertToBlockUnderConstruction( BlockUCState s, DatanodeStorageInfo[] targets) { if(isComplete()) { BlockInfoContiguousUnderConstruction ucBlock = new BlockInfoContiguousUnderConstruction(this, getBlockCollection().getBlockReplication(), s, targets); ucBlock.setBlockCollection(getBlockCollection()); return ucBlock; } // the block is already under construction BlockInfoContiguousUnderConstruction ucBlock = (BlockInfoContiguousUnderConstruction)this; ucBlock.setBlockUCState(s); ucBlock.setExpectedLocations(targets); ucBlock.setBlockCollection(getBlockCollection()); return ucBlock; }
BlockCollection bc, NumberReplicas numberReplicas) { final int numExpected = bc.getBlockReplication(); final int numLive = numberReplicas.liveReplicas(); if (numLive >= numExpected
short expectedReplication = bc.getBlockReplication(); NumberReplicas num = countNodes(block); int numCurrentReplica = num.liveReplicas();
short expectedReplication = bc.getBlockReplication(); NumberReplicas num = countNodes(block); int numCurrentReplica = num.liveReplicas();