public static BlockCommandProto convert(BlockCommand cmd) { BlockCommandProto.Builder builder = BlockCommandProto.newBuilder() .setBlockPoolId(cmd.getBlockPoolId()); switch (cmd.getAction()) { case DatanodeProtocol.DNA_TRANSFER: builder.setAction(BlockCommandProto.Action.TRANSFER); break; case DatanodeProtocol.DNA_INVALIDATE: builder.setAction(BlockCommandProto.Action.INVALIDATE); break; case DatanodeProtocol.DNA_SHUTDOWN: builder.setAction(BlockCommandProto.Action.SHUTDOWN); break; default: throw new AssertionError("Invalid action"); } Block[] blocks = cmd.getBlocks(); for (int i = 0; i < blocks.length; i++) { builder.addBlocks(PBHelperClient.convert(blocks[i])); } builder.addAllTargets(PBHelperClient.convert(cmd.getTargets())) .addAllTargetStorageUuids(convert(cmd.getTargetStorageIDs())); StorageType[][] types = cmd.getTargetStorageTypes(); if (types != null) { builder.addAllTargetStorageTypes(PBHelperClient.convert(types)); } return builder.build(); }
throw new AssertionError("Unknown action type: " + blkCmd.getAction()); return new BlockCommand(action, blkCmd.getBlockPoolId(), blocks, targets, targetStorageTypes, targetStorageIDs);
StorageType[][] storageTypes = {{StorageType.DEFAULT}, {StorageType.DEFAULT, StorageType.DEFAULT}}; BlockCommand bc = new BlockCommand(DatanodeProtocol.DNA_TRANSFER, "bp1", blocks, dnInfos, storageTypes, storageIDs); BlockCommandProto bcProto = PBHelper.convert(bc); BlockCommand bc2 = PBHelper.convert(bcProto); assertEquals(bc.getAction(), bc2.getAction()); assertEquals(bc.getBlocks().length, bc2.getBlocks().length); Block[] blocks2 = bc2.getBlocks(); for (int i = 0; i < blocks.length; i++) { assertEquals(blocks[i], blocks2[i]); DatanodeInfo[][] dnInfos2 = bc2.getTargets(); assertEquals(dnInfos.length, dnInfos2.length); for (int i = 0; i < dnInfos.length; i++) {
/** * Send a heartbeat to the name-node and replicate blocks if requested. */ @SuppressWarnings("unused") // keep it for future blockReceived benchmark int replicateBlocks() throws IOException { // register datanode DatanodeCommand[] cmds = nameNode.sendHeartbeat( dnRegistration, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, 0, 0, 0); if (cmds != null) { for (DatanodeCommand cmd : cmds) { if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) { // Send a copy of a block to another datanode BlockCommand bcmd = (BlockCommand)cmd; return transferBlocks(bcmd.getBlocks(), bcmd.getTargets()); } } } return 0; }
/** * Send a heartbeat to the name-node and replicate blocks if requested. */ @SuppressWarnings("unused") // keep it for future blockReceived benchmark int replicateBlocks() throws IOException { // register datanode StorageReport[] rep = { new StorageReport(storage, false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0L) }; DatanodeCommand[] cmds = dataNodeProto.sendHeartbeat(dnRegistration, rep, 0L, 0L, 0, 0, 0, null).getCommands(); if (cmds != null) { for (DatanodeCommand cmd : cmds) { if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) { // Send a copy of a block to another datanode BlockCommand bcmd = (BlockCommand)cmd; return transferBlocks(bcmd.getBlocks(), bcmd.getTargets(), bcmd.getTargetStorageIDs()); } } } return 0; }
/** * Test that getInvalidateBlocks observes the maxlimit. */ public void testGetInvalidateBlocks() throws Exception { final int MAX_BLOCKS = 10; final int REMAINING_BLOCKS = 2; final int MAX_LIMIT = MAX_BLOCKS - REMAINING_BLOCKS; DatanodeDescriptor dd = new DatanodeDescriptor(); ArrayList<Block> blockList = new ArrayList<Block>(MAX_BLOCKS); for (int i=0; i<MAX_BLOCKS; i++) { blockList.add(new Block(i, 0, GenerationStamp.FIRST_VALID_STAMP)); } dd.addBlocksToBeInvalidated(blockList); BlockCommand bc = dd.getInvalidateBlocks(MAX_LIMIT); assertEquals(bc.getBlocks().length, MAX_LIMIT); bc = dd.getInvalidateBlocks(MAX_LIMIT); assertEquals(bc.getBlocks().length, REMAINING_BLOCKS); }
case DatanodeProtocol.DNA_TRANSFER: transferBlocks(bcmd.getBlocks(), bcmd.getTargets()); myMetrics.blocksReplicated.inc(bcmd.getBlocks().length); break; case DatanodeProtocol.DNA_INVALIDATE: Block toDelete[] = bcmd.getBlocks(); try { if (blockScanner != null) { break; case DatanodeProtocol.DNA_RECOVERBLOCK: recoverBlocks(bcmd.getBlocks(), bcmd.getTargets()); break; default:
assertEquals(1, cmds.length); assertEquals(DatanodeProtocol.DNA_TRANSFER, cmds[0].getAction()); assertEquals(MAX_REPLICATE_LIMIT, ((BlockCommand)cmds[0]).getBlocks().length); assertEquals(2, cmds.length); assertEquals(DatanodeProtocol.DNA_TRANSFER, cmds[0].getAction()); assertEquals(MAX_REPLICATE_LIMIT, ((BlockCommand)cmds[0]).getBlocks().length); assertEquals(DatanodeProtocol.DNA_INVALIDATE, cmds[1].getAction()); assertEquals(MAX_INVALIDATE_LIMIT, ((BlockCommand)cmds[1]).getBlocks().length); assertEquals(2, cmds.length); assertEquals(DatanodeProtocol.DNA_TRANSFER, cmds[0].getAction()); assertEquals(REMAINING_BLOCKS, ((BlockCommand)cmds[0]).getBlocks().length); assertEquals(DatanodeProtocol.DNA_INVALIDATE, cmds[1].getAction()); assertEquals(MAX_INVALIDATE_LIMIT, ((BlockCommand)cmds[1]).getBlocks().length); assertEquals(1, cmds.length); assertEquals(DatanodeProtocol.DNA_INVALIDATE, cmds[0].getAction()); assertEquals(REMAINING_BLOCKS, ((BlockCommand)cmds[0]).getBlocks().length);
case DatanodeProtocol.DNA_TRANSFER: dn.transferBlocks(bcmd.getBlockPoolId(), bcmd.getBlocks(), bcmd.getTargets(), bcmd.getTargetStorageTypes(), bcmd.getTargetStorageIDs()); break; case DatanodeProtocol.DNA_INVALIDATE: Block toDelete[] = bcmd.getBlocks(); try { dn.getFSDataset().invalidate(bcmd.getBlockPoolId(), toDelete); } catch(IOException e) {
bcmd.getBlocks(), bcmd.getTargets()); myMetrics.blocksReplicated.inc(bcmd.getBlocks().length); break; case DatanodeProtocol.DNA_INVALIDATE: Block toDelete[] = bcmd.getBlocks(); try { if (blockScanner != null) { break; case DatanodeProtocol.DNA_RECOVERBLOCK: recoverBlocks(namespaceId, bcmd.getBlocks(), bcmd.getTargets()); break; default:
numReplicationTasks); if (pendingList != null && !pendingList.isEmpty()) { cmds.add(new BlockCommand(DatanodeProtocol.DNA_TRANSFER, blockPoolId, pendingList)); cmds.add(new BlockCommand(DatanodeProtocol.DNA_INVALIDATE, blockPoolId, blks));
assertEquals(1, cmds.length); assertEquals(DatanodeProtocol.DNA_TRANSFER, cmds[0].getAction()); assertEquals(MAX_REPLICATE_LIMIT, ((BlockCommand)cmds[0]).getBlocks().length); assertEquals(2, cmds.length); assertEquals(DatanodeProtocol.DNA_TRANSFER, cmds[0].getAction()); assertEquals(MAX_REPLICATE_LIMIT, ((BlockCommand)cmds[0]).getBlocks().length); assertEquals(DatanodeProtocol.DNA_INVALIDATE, cmds[1].getAction()); assertEquals(MAX_INVALIDATE_LIMIT, ((BlockCommand)cmds[1]).getBlocks().length); assertEquals(2, cmds.length); assertEquals(DatanodeProtocol.DNA_TRANSFER, cmds[0].getAction()); assertEquals(REMAINING_BLOCKS, ((BlockCommand)cmds[0]).getBlocks().length); assertEquals(DatanodeProtocol.DNA_INVALIDATE, cmds[1].getAction()); assertEquals(MAX_INVALIDATE_LIMIT, ((BlockCommand)cmds[1]).getBlocks().length); assertEquals(1, cmds.length); assertEquals(DatanodeProtocol.DNA_INVALIDATE, cmds[0].getAction()); assertEquals(REMAINING_BLOCKS, ((BlockCommand)cmds[0]).getBlocks().length);
public static BlockCommandProto convert(BlockCommand cmd) { BlockCommandProto.Builder builder = BlockCommandProto.newBuilder() .setBlockPoolId(cmd.getBlockPoolId()); switch (cmd.getAction()) { case DatanodeProtocol.DNA_TRANSFER: builder.setAction(BlockCommandProto.Action.TRANSFER); break; case DatanodeProtocol.DNA_INVALIDATE: builder.setAction(BlockCommandProto.Action.INVALIDATE); break; case DatanodeProtocol.DNA_SHUTDOWN: builder.setAction(BlockCommandProto.Action.SHUTDOWN); break; default: throw new AssertionError("Invalid action"); } Block[] blocks = cmd.getBlocks(); for (int i = 0; i < blocks.length; i++) { builder.addBlocks(PBHelper.convert(blocks[i])); } builder.addAllTargets(convert(cmd.getTargets())) .addAllTargetStorageUuids(convert(cmd.getTargetStorageIDs())); StorageType[][] types = cmd.getTargetStorageTypes(); if (types != null) { builder.addAllTargetStorageTypes(convert(types)); } return builder.build(); }
case DatanodeProtocol.DNA_TRANSFER: transferBlocks(bcmd.getBlocks(), bcmd.getTargets()); myMetrics.incrBlocksReplicated(bcmd.getBlocks().length); break; case DatanodeProtocol.DNA_INVALIDATE: Block toDelete[] = bcmd.getBlocks(); try { if (blockScanner != null) { break; case DatanodeProtocol.DNA_RECOVERBLOCK: recoverBlocks(bcmd.getBlocks(), bcmd.getTargets()); break; case DatanodeProtocol.DNA_ACCESSKEYUPDATE:
public Writable newInstance() { return new BlockCommand(); } });
public static BlockCommandProto convert(BlockCommand cmd) { BlockCommandProto.Builder builder = BlockCommandProto.newBuilder() .setBlockPoolId(cmd.getBlockPoolId()); switch (cmd.getAction()) { case DatanodeProtocol.DNA_TRANSFER: builder.setAction(BlockCommandProto.Action.TRANSFER); break; case DatanodeProtocol.DNA_INVALIDATE: builder.setAction(BlockCommandProto.Action.INVALIDATE); break; case DatanodeProtocol.DNA_SHUTDOWN: builder.setAction(BlockCommandProto.Action.SHUTDOWN); break; default: throw new AssertionError("Invalid action"); } Block[] blocks = cmd.getBlocks(); for (int i = 0; i < blocks.length; i++) { builder.addBlocks(PBHelper.convert(blocks[i])); } builder.addAllTargets(convert(cmd.getTargets())) .addAllTargetStorageUuids(convert(cmd.getTargetStorageIDs())); StorageType[][] types = cmd.getTargetStorageTypes(); if (types != null) { builder.addAllTargetStorageTypes(convert(types)); } return builder.build(); }
public Writable newInstance() { return new BlockCommand(); } });
case DatanodeProtocol.DNA_TRANSFER: dn.transferBlocks(bcmd.getBlockPoolId(), bcmd.getBlocks(), bcmd.getTargets(), bcmd.getTargetStorageTypes()); dn.metrics.incrBlocksReplicated(bcmd.getBlocks().length); break; case DatanodeProtocol.DNA_INVALIDATE: Block toDelete[] = bcmd.getBlocks(); try { dn.getFSDataset().invalidate(bcmd.getBlockPoolId(), toDelete); } catch(IOException e) {
BlockCommand getLeaseRecoveryCommand(int maxTransfers) { List<BlockTargetPair> blocktargetlist = recoverBlocks.poll(maxTransfers); return blocktargetlist == null? null: new BlockCommand(DatanodeProtocol.DNA_RECOVERBLOCK, blocktargetlist); }
case DatanodeProtocol.DNA_TRANSFER: dn.transferBlocks(bcmd.getBlockPoolId(), bcmd.getBlocks(), bcmd.getTargets(), bcmd.getTargetStorageTypes()); dn.metrics.incrBlocksReplicated(bcmd.getBlocks().length); break; case DatanodeProtocol.DNA_INVALIDATE: Block toDelete[] = bcmd.getBlocks(); try { dn.getFSDataset().invalidate(bcmd.getBlockPoolId(), toDelete); } catch(IOException e) {