for (; i < numBlocks - 1; i++) { blk.readFields(in); blocksContiguous[i] = new BlockInfoContiguous(blk, blockReplication); blocksContiguous[i] = new BlockInfoContiguous(blk, blockReplication); blocksContiguous[i].convertToBlockUnderConstruction( BlockUCState.UNDER_CONSTRUCTION, null);
blocks[j] = new BlockInfoContiguous(replication); blocks[j].readFields(in);
blockInfo = new BlockInfoContiguous(block, numLocations); blockInfo.convertToBlockUnderConstruction( HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets);
newBI = new BlockInfoStriped(newBlock, ecPolicy); } else { newBI = new BlockInfoContiguous(newBlock, file.getPreferredBlockReplication()); newBI = new BlockInfoStriped(newBlock, ecPolicy); } else { newBI = new BlockInfoContiguous(newBlock, file.getFileReplication());
BlockToMarkCorrupt(BlockInfoContiguous stored, long gs, String reason, Reason reasonCode) { this(new BlockInfoContiguous(stored), stored, reason, reasonCode); //the corrupted block in datanode has a different generation stamp corrupted.setGenerationStamp(gs); }
BlockToMarkCorrupt(BlockInfoContiguous stored, long gs, String reason, Reason reasonCode) { this(new BlockInfoContiguous(stored), stored, reason, reasonCode); //the corrupted block in datanode has a different generation stamp corrupted.setGenerationStamp(gs); }
newBlockInfo = new BlockInfoStriped(newBlock, ecPolicy); } else { newBlockInfo = new BlockInfoContiguous(newBlock, file.getPreferredBlockReplication());
/** * Convert an under construction block to a complete block. * * @return BlockInfo - a complete block. * @throws IOException if the state of the block * (the generation stamp and the length) has not been committed by * the client or it does not have at least a minimal number of replicas * reported from data-nodes. */ BlockInfoContiguous convertToCompleteBlock() throws IOException { assert getBlockUCState() != BlockUCState.COMPLETE : "Trying to convert a COMPLETE block"; return new BlockInfoContiguous(this); }
/** * Convert an under construction block to a complete block. * * @return BlockInfo - a complete block. * @throws IOException if the state of the block * (the generation stamp and the length) has not been committed by * the client or it does not have at least a minimal number of replicas * reported from data-nodes. */ BlockInfoContiguous convertToCompleteBlock() throws IOException { assert getBlockUCState() != BlockUCState.COMPLETE : "Trying to convert a COMPLETE block"; return new BlockInfoContiguous(this); }
blocks[i] = new BlockInfoStriped(PBHelperClient.convert(b), ecPolicy); } else { blocks[i] = new BlockInfoContiguous(PBHelperClient.convert(b), replication); ucBlk = new BlockInfoStriped(striped, ecPolicy); } else { ucBlk = new BlockInfoContiguous(lastBlk, replication);
truncatedBlockUC = new BlockInfoContiguous(newBlock, file.getPreferredBlockReplication()); truncatedBlockUC.convertToBlockUnderConstruction( assert !oldBlock.isComplete() : "oldBlock should be under construction"; BlockUnderConstructionFeature uc = oldBlock.getUnderConstructionFeature(); uc.setTruncateBlock(new BlockInfoContiguous(oldBlock, oldBlock.getReplication())); uc.getTruncateBlock().setNumBytes(oldBlock.getNumBytes() - lastBlockDelta);
if(storedBlock == null) { storedBlock = (BlockInfoContiguous) fsn.getBlockManager() .addBlockCollectionWithCheck(new BlockInfoContiguous(blk, copy.getFileReplication()), file);
private BlockInfoContiguous addBlockToBM(long blkId) { Block block = new Block(blkId); BlockInfoContiguous blockInfo = new BlockInfoContiguous(block, (short) 3); BlockCollection bc = Mockito.mock(BlockCollection.class); Mockito.doReturn((short) 3).when(bc).getBlockReplication(); bm.blocksMap.addBlockCollection(blockInfo, bc); return blockInfo; }
private BlockInfoContiguous blockOnNodes(long blkId, List<DatanodeDescriptor> nodes) { Block block = new Block(blkId); BlockInfoContiguous blockInfo = new BlockInfoContiguous(block, (short) 3); for (DatanodeDescriptor dn : nodes) { for (DatanodeStorageInfo storage : dn.getStorageInfos()) { blockInfo.addStorage(storage); } } return blockInfo; }
/** * Creates the required number of files with one block each * @param nCount Number of INodes to create * @return Array of INode files */ private INodeFile[] createINodeFiles(int nCount, String fileNamePrefix) { if(nCount <= 0) return new INodeFile[1]; replication = 3; preferredBlockSize = 128 * 1024 * 1024; INodeFile[] iNodes = new INodeFile[nCount]; for (int i = 0; i < nCount; i++) { iNodes[i] = new INodeFile(i, null, perm, 0L, 0L, null, replication, preferredBlockSize, (byte)0); iNodes[i].setLocalName(DFSUtil.string2Bytes(fileNamePrefix + i)); BlockInfoContiguous newblock = new BlockInfoContiguous(replication); iNodes[i].addBlock(newblock); } return iNodes; }
@Test public void testCommitBlockSynchronization() throws IOException { INodeFile file = mockFileUnderConstruction(); Block block = new Block(blockId, length, genStamp); FSNamesystem namesystemSpy = makeNameSystemSpy(block, file); DatanodeID[] newTargets = new DatanodeID[0]; ExtendedBlock lastBlock = new ExtendedBlock(); namesystemSpy.commitBlockSynchronization( lastBlock, genStamp, length, false, false, newTargets, null); // Repeat the call to make sure it does not throw namesystemSpy.commitBlockSynchronization( lastBlock, genStamp, length, false, false, newTargets, null); // Simulate 'completing' the block. BlockInfoContiguous completedBlockInfo = new BlockInfoContiguous(block, (short) 1); completedBlockInfo.setBlockCollection(file); completedBlockInfo.setGenerationStamp(genStamp); doReturn(completedBlockInfo).when(namesystemSpy) .getStoredBlock(any(Block.class)); doReturn(completedBlockInfo).when(file).getLastBlock(); // Repeat the call to make sure it does not throw namesystemSpy.commitBlockSynchronization( lastBlock, genStamp, length, false, false, newTargets, null); }
@Test public void testCommitBlockSynchronizationWithClose() throws IOException { INodeFile file = mockFileUnderConstruction(); Block block = new Block(blockId, length, genStamp); FSNamesystem namesystemSpy = makeNameSystemSpy(block, file); DatanodeID[] newTargets = new DatanodeID[0]; ExtendedBlock lastBlock = new ExtendedBlock(); namesystemSpy.commitBlockSynchronization( lastBlock, genStamp, length, true, false, newTargets, null); // Repeat the call to make sure it returns true namesystemSpy.commitBlockSynchronization( lastBlock, genStamp, length, true, false, newTargets, null); BlockInfoContiguous completedBlockInfo = new BlockInfoContiguous(block, (short) 1); completedBlockInfo.setBlockCollection(file); completedBlockInfo.setGenerationStamp(genStamp); doReturn(completedBlockInfo).when(namesystemSpy) .getStoredBlock(any(Block.class)); doReturn(completedBlockInfo).when(file).getLastBlock(); namesystemSpy.commitBlockSynchronization( lastBlock, genStamp, length, true, false, newTargets, null); }
@Test public void testAddStorage() throws Exception { BlockInfoContiguous blockInfo = new BlockInfoContiguous((short) 3); final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo("storageID", "127.0.0.1"); boolean added = blockInfo.addStorage(storage); Assert.assertTrue(added); Assert.assertEquals(storage, blockInfo.getStorageInfo(0)); }
@Test public void testReplaceStorage() throws Exception { // Create two dummy storages. final DatanodeStorageInfo storage1 = DFSTestUtil.createDatanodeStorageInfo("storageID1", "127.0.0.1"); final DatanodeStorageInfo storage2 = new DatanodeStorageInfo(storage1.getDatanodeDescriptor(), new DatanodeStorage("storageID2")); final int NUM_BLOCKS = 10; BlockInfoContiguous[] blockInfos = new BlockInfoContiguous[NUM_BLOCKS]; // Create a few dummy blocks and add them to the first storage. for (int i = 0; i < NUM_BLOCKS; ++i) { blockInfos[i] = new BlockInfoContiguous((short) 3); storage1.addBlock(blockInfos[i]); } // Try to move one of the blocks to a different storage. boolean added = storage2.addBlock(blockInfos[NUM_BLOCKS / 2]) == AddBlockResult.ADDED; Assert.assertThat(added, is(false)); Assert.assertThat(blockInfos[NUM_BLOCKS/2].getStorageInfo(0), is(storage2)); }
@Test public void testBlocksCounter() throws Exception { DatanodeDescriptor dd = BlockManagerTestUtil.getLocalDatanodeDescriptor(true); assertEquals(0, dd.numBlocks()); BlockInfoContiguous blk = new BlockInfoContiguous(new Block(1L), (short) 1); BlockInfoContiguous blk1 = new BlockInfoContiguous(new Block(2L), (short) 2); DatanodeStorageInfo[] storages = dd.getStorageInfos(); assertTrue(storages.length > 0); // add first block assertTrue(storages[0].addBlock(blk) == AddBlockResult.ADDED); assertEquals(1, dd.numBlocks()); // remove a non-existent block assertFalse(dd.removeBlock(blk1)); assertEquals(1, dd.numBlocks()); // add an existent block assertFalse(storages[0].addBlock(blk) == AddBlockResult.ADDED); assertEquals(1, dd.numBlocks()); // add second block assertTrue(storages[0].addBlock(blk1) == AddBlockResult.ADDED); assertEquals(2, dd.numBlocks()); // remove first block assertTrue(dd.removeBlock(blk)); assertEquals(1, dd.numBlocks()); // remove second block assertTrue(dd.removeBlock(blk1)); assertEquals(0, dd.numBlocks()); } }