private static void logRecoverBlock(String who, RecoveringBlock rb) { ExtendedBlock block = rb.getBlock(); DatanodeInfo[] targets = rb.getLocations(); LOG.info(who + " calls recoverBlock(" + block + ", targets=[" + Joiner.on(", ").join(targets) + "]" + ", newGenerationStamp=" + rb.getNewGenerationStamp() + ")"); }
private Collection<RecoveringBlock> initRecoveringBlocks() { Collection<RecoveringBlock> blocks = new ArrayList<RecoveringBlock>(1); DatanodeInfo[] locs = new DatanodeInfo[] { new DatanodeInfo(dn.dnRegistration), mock(DatanodeInfo.class) }; RecoveringBlock rBlock = new RecoveringBlock(block, locs, RECOVERY_ID); blocks.add(rBlock); return blocks; } /**
getActiveNamenodeForBP(block.getBlockPoolId()); boolean isTruncateRecovery = rBlock.getNewBlock() != null; long blockId = (isTruncateRecovery) ? rBlock.getNewBlock().getBlockId() : block.getBlockId(); newBlock.setNumBytes(rBlock.getNewBlock().getNumBytes());
assertEquals(1, recoveryCommand.getRecoveringBlocks().size()); DatanodeInfo[] recoveringNodes = recoveryCommand.getRecoveringBlocks() .toArray(new BlockRecoveryCommand.RecoveringBlock[0])[0].getLocations(); assertEquals(3, recoveringNodes.length); assertEquals(recoveringNodes[0], dd1); assertEquals(1, recoveryCommand.getRecoveringBlocks().size()); recoveringNodes = recoveryCommand.getRecoveringBlocks() .toArray(new BlockRecoveryCommand.RecoveringBlock[0])[0].getLocations(); assertEquals(2, recoveringNodes.length); assertEquals(1, recoveryCommand.getRecoveringBlocks().size()); recoveringNodes = recoveryCommand.getRecoveringBlocks() .toArray(new BlockRecoveryCommand.RecoveringBlock[0])[0].getLocations();
ExtendedBlock block = rBlock.getBlock(); final String bpid = block.getBlockPoolId(); DatanodeProtocolClientSideTranslatorPB nn = getActiveNamenodeForBP(block.getBlockPoolId()); long recoveryId = rBlock.getNewGenerationStamp(); boolean isTruncateRecovery = rBlock.getNewBlock() != null; long blockId = (isTruncateRecovery) ? rBlock.getNewBlock().getBlockId() : block.getBlockId(); newBlock.setNumBytes(rBlock.getNewBlock().getNumBytes());
Block recoveryBlock = (copyOnTruncateRecovery) ? b : b.getTruncateBlock(); brCommand.add(new RecoveringBlock(primaryBlock, recoveryInfos, recoveryBlock)); } else { brCommand.add(new RecoveringBlock(primaryBlock, recoveryInfos, b.getBlockRecoveryId()));
Block recoveryBlock = (copyOnTruncateRecovery) ? b : b.getTruncateBlock(); brCommand.add(new RecoveringBlock(primaryBlock, recoveryInfos, recoveryBlock)); } else { brCommand.add(new RecoveringBlock(primaryBlock, recoveryInfos, b.getBlockRecoveryId()));
ExtendedBlock block = rBlock.getBlock(); final String bpid = block.getBlockPoolId(); DatanodeProtocolClientSideTranslatorPB nn = getActiveNamenodeForBP(block.getBlockPoolId()); long recoveryId = rBlock.getNewGenerationStamp(); boolean isTruncateRecovery = rBlock.getNewBlock() != null; long blockId = (isTruncateRecovery) ? rBlock.getNewBlock().getBlockId() : block.getBlockId(); newBlock.setNumBytes(rBlock.getNewBlock().getNumBytes());
long recoveryId = b.getGenerationStamp() + 1; idp.initReplicaRecovery( new RecoveringBlock(b, locatedblock.getLocations(), recoveryId)); b.getBlockId(), 0, 0); assertNull(idp.initReplicaRecovery( new RecoveringBlock(badBlock, locatedblock.getLocations(), recoveryId)));
final long newlength = b.getNumBytes() - 1; final ReplicaRecoveryInfo rri = fsdataset.initReplicaRecovery( new RecoveringBlock(b, null, recoveryid));
final FsDatasetSpi<?> fsdataset = DataNodeTestUtils.getFSDataset(datanode); final ReplicaRecoveryInfo rri = fsdataset.initReplicaRecovery( new RecoveringBlock(b, null, recoveryid));
long recoveryId = b.getGenerationStamp() + 1; idp.initReplicaRecovery( new RecoveringBlock(b, locatedblock.getLocations(), recoveryId));
ExtendedBlock block = rBlock.getBlock(); String blookPoolId = block.getBlockPoolId(); DatanodeID[] datanodeids = rBlock.getLocations(); List<BlockRecord> syncList = new ArrayList<BlockRecord>(datanodeids.length); int errorCount = 0; "Recovery for replica " + block + " on data-node " + id + " is already in progress. Recovery id = " + rBlock.getNewGenerationStamp() + " is aborted.", ripE); return; } catch (IOException e) {
ExtendedBlock block = rBlock.getBlock(); String blookPoolId = block.getBlockPoolId(); DatanodeID[] datanodeids = rBlock.getLocations(); List<BlockRecord> syncList = new ArrayList<BlockRecord>(datanodeids.length); int errorCount = 0; "Recovery for replica " + block + " on data-node " + id + " is already in progress. Recovery id = " + rBlock.getNewGenerationStamp() + " is aborted.", ripE); return; } catch (IOException e) {
DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512)); streams.getChecksumOut().write('a'); dn.data.initReplicaRecovery(new RecoveringBlock(block, null, RECOVERY_ID+1)); try { dn.syncBlock(rBlock, initBlockRecords(dn));
/** * BlockRecoveryFI_11. a replica's recovery id does not match new GS. * * @throws IOException in case of an error */ @Test public void testNotMatchedReplicaID() throws IOException { if(LOG.isDebugEnabled()) { LOG.debug("Running " + GenericTestUtils.getMethodName()); } ReplicaInPipelineInterface replicaInfo = dn.data.createRbw(block); BlockWriteStreams streams = null; try { streams = replicaInfo.createStreams(true, 0, 0); streams.checksumOut.write('a'); dn.data.initReplicaRecovery(new RecoveringBlock(block, null, RECOVERY_ID+1)); try { dn.syncBlock(rBlock, initBlockRecords(dn)); fail("Sync should fail"); } catch (IOException e) { e.getMessage().startsWith("Cannot recover "); } verify(dn.namenode, never()).commitBlockSynchronization( any(Block.class), anyLong(), anyLong(), anyBoolean(), anyBoolean(), any(DatanodeID[].class)); } finally { streams.close(); } } }
/** Test to verify that InterDatanode RPC timesout as expected when * the server DN does not respond. */ @Test(expected=SocketTimeoutException.class) public void testInterDNProtocolTimeout() throws Throwable { final Server server = new TestServer(1, true); server.start(); final InetSocketAddress addr = NetUtils.getConnectAddress(server); DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort()); DatanodeInfo dInfo = new DatanodeInfo(fakeDnId); InterDatanodeProtocol proxy = null; try { proxy = DataNode.createInterDataNodeProtocolProxy( dInfo, conf, 500, false); proxy.initReplicaRecovery(new RecoveringBlock( new ExtendedBlock("bpid", 1), null, 100)); fail ("Expected SocketTimeoutException exception, but did not get."); } finally { if (proxy != null) { RPC.stopProxy(proxy); } server.stop(); } } }
/** Sync two replicas */ private void testSyncReplicas(ReplicaRecoveryInfo replica1, ReplicaRecoveryInfo replica2, InterDatanodeProtocol dn1, InterDatanodeProtocol dn2, long expectLen) throws IOException { DatanodeInfo[] locs = new DatanodeInfo[]{ mock(DatanodeInfo.class), mock(DatanodeInfo.class)}; RecoveringBlock rBlock = new RecoveringBlock(block, locs, RECOVERY_ID); ArrayList<BlockRecord> syncList = new ArrayList<BlockRecord>(2); BlockRecord record1 = new BlockRecord( new DatanodeID("xx", "yy", 44, 55), dn1, replica1); BlockRecord record2 = new BlockRecord( new DatanodeID("aa", "bb", 11, 22), dn2, replica2); syncList.add(record1); syncList.add(record2); when(dn1.updateReplicaUnderRecovery((Block)anyObject(), anyLong(), anyLong())).thenReturn(new Block(block.getBlockId(), expectLen, block.getGenerationStamp())); when(dn2.updateReplicaUnderRecovery((Block)anyObject(), anyLong(), anyLong())).thenReturn(new Block(block.getBlockId(), expectLen, block.getGenerationStamp())); dn.syncBlock(rBlock, syncList); }
/** Sync two replicas */ private void testSyncReplicas(ReplicaRecoveryInfo replica1, ReplicaRecoveryInfo replica2, InterDatanodeProtocol dn1, InterDatanodeProtocol dn2, long expectLen) throws IOException { DatanodeInfo[] locs = new DatanodeInfo[]{ mock(DatanodeInfo.class), mock(DatanodeInfo.class)}; RecoveringBlock rBlock = new RecoveringBlock(block, locs, RECOVERY_ID); ArrayList<BlockRecord> syncList = new ArrayList<BlockRecord>(2); BlockRecord record1 = new BlockRecord( DFSTestUtil.getDatanodeInfo("1.2.3.4", "bogus", 1234), dn1, replica1); BlockRecord record2 = new BlockRecord( DFSTestUtil.getDatanodeInfo("1.2.3.4", "bogus", 1234), dn2, replica2); syncList.add(record1); syncList.add(record2); when(dn1.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(), anyLong(), anyLong())).thenReturn("storage1"); when(dn2.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(), anyLong(), anyLong())).thenReturn("storage2"); dn.syncBlock(rBlock, syncList); }
@Test public void testConvertBlockRecoveryCommand() { DatanodeInfo di1 = DFSTestUtil.getLocalDatanodeInfo(); DatanodeInfo di2 = DFSTestUtil.getLocalDatanodeInfo(); DatanodeInfo[] dnInfo = new DatanodeInfo[] { di1, di2 }; List<RecoveringBlock> blks = ImmutableList.of( new RecoveringBlock(getExtendedBlock(1), dnInfo, 3), new RecoveringBlock(getExtendedBlock(2), dnInfo, 3) ); BlockRecoveryCommand cmd = new BlockRecoveryCommand(blks); BlockRecoveryCommandProto proto = PBHelper.convert(cmd); assertEquals(1, proto.getBlocks(0).getBlock().getB().getBlockId()); assertEquals(2, proto.getBlocks(1).getBlock().getB().getBlockId()); BlockRecoveryCommand cmd2 = PBHelper.convert(proto); List<RecoveringBlock> cmd2Blks = Lists.newArrayList( cmd2.getRecoveringBlocks()); assertEquals(blks.get(0).getBlock(), cmd2Blks.get(0).getBlock()); assertEquals(blks.get(1).getBlock(), cmd2Blks.get(1).getBlock()); assertEquals(Joiner.on(",").join(blks), Joiner.on(",").join(cmd2Blks)); assertEquals(cmd.toString(), cmd2.toString()); }