storageTypes[i] = s.getStorageType(); BlockWithLocations blkWithLocs = new BlockWithLocations(block, datanodeUuids, storageIDs, storageTypes); if(block.isStriped()) {
for (BlockWithLocations blkLocs : newBlksLocs.getBlocks()) { if (blkLocs.getBlock().getNumBytes() < getBlocksMinBlockSize) { continue; sblkLocs.getDataBlockNum(), sblkLocs.getCellSize()); } else { bytesReceived += blkLocs.getBlock().getNumBytes(); block = new DBlock(blkLocs.getBlock()); block = globalBlocks.putIfAbsent(blkLocs.getBlock(), block); synchronized (block) { block.clearLocations(); final String[] datanodeUuids = blkLocs.getDatanodeUuids(); final StorageType[] storageTypes = blkLocs.getStorageTypes(); for (int i = 0; i < datanodeUuids.length; i++) { final StorageGroup g = storageGroupMap.get(
public static BlockWithLocationsProto convert(BlockWithLocations blk) { BlockWithLocationsProto.Builder builder = BlockWithLocationsProto .newBuilder().setBlock(PBHelperClient.convert(blk.getBlock())) .addAllDatanodeUuids(Arrays.asList(blk.getDatanodeUuids())) .addAllStorageUuids(Arrays.asList(blk.getStorageIDs())) .addAllStorageTypes(PBHelperClient.convertStorageTypes(blk.getStorageTypes())); if (blk instanceof StripedBlockWithLocations) { StripedBlockWithLocations sblk = (StripedBlockWithLocations) blk; builder.setIndices(PBHelperClient.getByteString(sblk.getIndices())); builder.setDataBlockNum(sblk.getDataBlockNum()); builder.setCellSize(sblk.getCellSize()); } return builder.build(); }
locs = namenode.getBlocks(dataNodes[0], fileLen).getBlocks(); assertEquals(locs.length, 12); assertEquals(locs[0].getStorageIDs().length, 2); assertEquals(locs[1].getStorageIDs().length, 2); assertEquals(locs[0].getStorageIDs().length, 2); assertEquals(locs[0].getStorageIDs().length, 2);
locs = namenode.getBlocks(dataNodes[0], fileLen).getBlocks(); assertEquals(locs.length, 2); assertEquals(locs[0].getDatanodes().length, 2); assertEquals(locs[1].getDatanodes().length, 2); assertEquals(locs[0].getDatanodes().length, 2); assertEquals(locs[0].getDatanodes().length, 2);
for (BlockWithLocations blk : newBlocks.getBlocks()) { if (blk.getBlock().getNumBytes() < getBlocksMinBlockSize) { continue; bytesReceived += blk.getBlock().getNumBytes(); synchronized (globalBlocks) { final DBlock block = globalBlocks.get(blk.getBlock()); synchronized (block) { block.clearLocations(); final String[] datanodeUuids = blk.getDatanodeUuids(); final StorageType[] storageTypes = blk.getStorageTypes(); for (int i = 0; i < datanodeUuids.length; i++) { final StorageGroup g = storageGroupMap.get(
for (BlockWithLocations blk : newBlocks.getBlocks()) { if (blk.getBlock().getNumBytes() < getBlocksMinBlockSize) { continue; bytesReceived += blk.getBlock().getNumBytes(); synchronized (globalBlocks) { final DBlock block = globalBlocks.get(blk.getBlock()); synchronized (block) { block.clearLocations(); final String[] datanodeUuids = blk.getDatanodeUuids(); final StorageType[] storageTypes = blk.getStorageTypes(); for (int i = 0; i < datanodeUuids.length; i++) { final StorageGroup g = storageGroupMap.get(
long bytesReceived = 0; for (BlockWithLocations blk : newBlocks) { bytesReceived += blk.getBlock().getNumBytes(); BalancerBlock block; synchronized(globalBlockList) { block = globalBlockList.get(blk.getBlock()); if (block==null) { block = new BalancerBlock(blk.getBlock()); globalBlockList.put(blk.getBlock(), block); } else { block.clearLocations(); for ( String location : blk.getDatanodes() ) { BalancerDatanode datanode = datanodes.get(location); if (datanode != null) { // not an unknown datanode
long bytesReceived = 0; for (BlockWithLocations blk : newBlocks) { bytesReceived += blk.getBlock().getNumBytes(); BalancerBlock block; synchronized(globalBlockList) { block = globalBlockList.get(blk.getBlock()); if (block==null) { block = new BalancerBlock(blk.getBlock()); globalBlockList.put(blk.getBlock(), block); } else { block.clearLocations(); for ( String location : blk.getDatanodes() ) { BalancerDatanode datanode = datanodes.get(location); if (datanode != null) { // not an unknown datanode
/** * Get all valid locations of the block & add the block to results * return the length of the added block; 0 if the block is not added */ private long addBlock(Block block, List<BlockWithLocations> results) { ArrayList<String> machineSet = new ArrayList<String>(blocksMap.numNodes(block)); for(Iterator<DatanodeDescriptor> it = blocksMap.nodeIterator(block); it.hasNext();) { String storageID = it.next().getStorageID(); // filter invalidate replicas Collection<Block> blocks = recentInvalidateSets.get(storageID); if(blocks==null || !blocks.contains(block)) { machineSet.add(storageID); } } if(machineSet.size() == 0) { return 0; } else { results.add(new BlockWithLocations(block, machineSet.toArray(new String[machineSet.size()]))); return block.getNumBytes(); } }
/** * Get all valid locations of the block & add the block to results * return the length of the added block; 0 if the block is not added */ private long addBlock(Block block, List<BlockWithLocations> results) { ArrayList<String> machineSet = new ArrayList<String>(blocksMap.numNodes(block)); for (Iterator<DatanodeDescriptor> it = blocksMap.nodeIterator(block); it.hasNext();) { String storageID = it.next().getStorageID(); // filter invalidate replicas LightWeightHashSet<Block> blocks = recentInvalidateSets.get(storageID); if (blocks == null || !blocks.contains(block)) { machineSet.add(storageID); } } if (machineSet.size() == 0) { return 0; } else { results.add(new BlockWithLocations(block, machineSet.toArray(new String[machineSet.size()]))); return block.getNumBytes(); } }
/** * Get all valid locations of the block & add the block to results * return the length of the added block; 0 if the block is not added */ private long addBlock(Block block, List<BlockWithLocations> results) { final List<DatanodeStorageInfo> locations = getValidLocations(block); if(locations.size() == 0) { return 0; } else { final String[] datanodeUuids = new String[locations.size()]; final String[] storageIDs = new String[datanodeUuids.length]; final StorageType[] storageTypes = new StorageType[datanodeUuids.length]; for(int i = 0; i < locations.size(); i++) { final DatanodeStorageInfo s = locations.get(i); datanodeUuids[i] = s.getDatanodeDescriptor().getDatanodeUuid(); storageIDs[i] = s.getStorageID(); storageTypes[i] = s.getStorageType(); } results.add(new BlockWithLocations(block, datanodeUuids, storageIDs, storageTypes)); return block.getNumBytes(); } }
/** * Get all valid locations of the block & add the block to results * return the length of the added block; 0 if the block is not added */ private long addBlock(Block block, List<BlockWithLocations> results) { final List<DatanodeStorageInfo> locations = getValidLocations(block); if(locations.size() == 0) { return 0; } else { final String[] datanodeUuids = new String[locations.size()]; final String[] storageIDs = new String[datanodeUuids.length]; final StorageType[] storageTypes = new StorageType[datanodeUuids.length]; for(int i = 0; i < locations.size(); i++) { final DatanodeStorageInfo s = locations.get(i); datanodeUuids[i] = s.getDatanodeDescriptor().getDatanodeUuid(); storageIDs[i] = s.getStorageID(); storageTypes[i] = s.getStorageType(); } results.add(new BlockWithLocations(block, datanodeUuids, storageIDs, storageTypes)); return block.getNumBytes(); } }
@Override public String toString() { final StringBuilder b = new StringBuilder(); b.append(block); if (datanodeUuids.length == 0) { return b.append("[]").toString(); } appendString(0, b.append("[")); for(int i = 1; i < datanodeUuids.length; i++) { appendString(i, b.append(",")); } return b.append("]").toString(); }
@Override public String toString() { final StringBuilder b = new StringBuilder(); b.append(block); if (datanodeUuids.length == 0) { return b.append("[]").toString(); } appendString(0, b.append("[")); for(int i = 1; i < datanodeUuids.length; i++) { appendString(i, b.append(",")); } return b.append("]").toString(); }
private static BlockWithLocations getBlockWithLocations(int bid) { final String[] datanodeUuids = {"dn1", "dn2", "dn3"}; final String[] storageIDs = {"s1", "s2", "s3"}; final StorageType[] storageTypes = { StorageType.DISK, StorageType.DISK, StorageType.DISK}; return new BlockWithLocations(new Block(bid, 0, 1), datanodeUuids, storageIDs, storageTypes); }
/** deserialization method */ public void readFields(DataInput in) throws IOException { int len = WritableUtils.readVInt(in); blocks = new BlockWithLocations[len]; for(int i=0; i<len; i++) { blocks[i] = new BlockWithLocations(); blocks[i].readFields(in); } } }
/** serialization method */ public void write( DataOutput out ) throws IOException { WritableUtils.writeVInt(out, blocks.length); for(int i=0; i<blocks.length; i++) { blocks[i].write(out); } }
/** serialization method */ public void write( DataOutput out ) throws IOException { WritableUtils.writeVInt(out, blocks.length); for(int i=0; i<blocks.length; i++) { blocks[i].write(out); } }
private void compare(BlockWithLocations locs1, BlockWithLocations locs2) { assertEquals(locs1.getBlock(), locs2.getBlock()); assertTrue(Arrays.equals(locs1.getStorageIDs(), locs2.getStorageIDs())); }