/** * getNodes function returns a list of DiskBalancerDataNodes. * * @return Array of DiskBalancerDataNodes */ @Override public List<DiskBalancerDataNode> getNodes() throws Exception { Preconditions.checkNotNull(this.connector); List<DiskBalancerDataNode> nodeList = new LinkedList<>(); DatanodeStorageReport[] reports = this.connector .getLiveDatanodeStorageReport(); for (DatanodeStorageReport report : reports) { DiskBalancerDataNode datanode = getBalancerNodeFromDataNode( report.getDatanodeInfo()); getVolumeInfoFromStorageReports(datanode, report.getStorageReports()); nodeList.add(datanode); } return nodeList; }
/** * Generates datanode reports for the given report type. * * @param type * type of the datanode report * @return array of DatanodeStorageReports */ public DatanodeStorageReport[] getDatanodeStorageReport( DatanodeReportType type) { final List<DatanodeDescriptor> datanodes = getDatanodeListForReport(type); DatanodeStorageReport[] reports = new DatanodeStorageReport[datanodes .size()]; for (int i = 0; i < reports.length; i++) { final DatanodeDescriptor d = datanodes.get(i); reports[i] = new DatanodeStorageReport( new DatanodeInfoBuilder().setFrom(d).build(), d.getStorageReports()); } return reports; } }
private static long getCapacity(DatanodeStorageReport report, StorageType t) { long capacity = 0L; for(StorageReport r : report.getStorageReports()) { if (r.getStorage().getStorageType() == t) { capacity += r.getCapacity(); } } return capacity; }
/** Get live datanode storage reports and then build the network topology. */ public List<DatanodeStorageReport> init() throws IOException { final DatanodeStorageReport[] reports = nnc.getLiveDatanodeStorageReport(); final List<DatanodeStorageReport> trimmed = new ArrayList<DatanodeStorageReport>(); // create network topology and classify utilization collections: // over-utilized, above-average, below-average and under-utilized. for (DatanodeStorageReport r : DFSUtil.shuffle(reports)) { final DatanodeInfo datanode = r.getDatanodeInfo(); if (shouldIgnore(datanode)) { continue; } trimmed.add(r); cluster.add(datanode); } return trimmed; }
private static long getMaxRemaining(DatanodeStorageReport report, StorageType t) { long max = 0L; for(StorageReport r : report.getStorageReports()) { if (r.getStorage().getStorageType() == t) { if (r.getRemaining() > max) { max = r.getRemaining(); } } } return max; }
final DDatanode dn = dispatcher.newDatanode(r.getDatanodeInfo()); final boolean isSource = Util.isIncluded(sourceNodes, dn.getDatanodeInfo()); for(StorageType t : StorageType.getMovableTypes()) {
.getLiveDatanodeStorageReport(); for (DatanodeStorageReport storage : liveDns) { StorageReport[] storageReports = storage.getStorageReports(); List<StorageType> storageTypes = new ArrayList<>(); List<Long> remainingSizeList = new ArrayList<>(); datanodeMap.addTarget(storage.getDatanodeInfo(), storageTypes, remainingSizeList);
private long getRemaining(DatanodeStorageReport report, StorageType t) { long remaining = 0L; for(StorageReport r : report.getStorageReports()) { if (r.getStorage().getStorageType() == t) { if (r.getRemaining() >= defaultBlockSize) { remaining += r.getRemaining(); } } } return remaining; }
void init() throws IOException { initStoragePolicies(); final List<DatanodeStorageReport> reports = dispatcher.init(); for(DatanodeStorageReport r : reports) { final DDatanode dn = dispatcher.newDatanode(r.getDatanodeInfo()); for(StorageType t : StorageType.getMovableTypes()) { final Source source = dn.addSource(t, Long.MAX_VALUE, dispatcher); final long maxRemaining = getMaxRemaining(r, t); final StorageGroup target = maxRemaining > 0L ? dn.addTarget(t, maxRemaining) : null; storages.add(source, target); } } }
public static DatanodeStorageReport convertDatanodeStorageReport( DatanodeStorageReportProto proto) { return new DatanodeStorageReport( convert(proto.getDatanodeInfo()), convertStorageReports(proto.getStorageReportsList())); }
public HashMap<String, Integer> getNumberOfDataDirsPerHost(){ HashMap<String, Integer> disksPerHost = new HashMap<>(); try { @SuppressWarnings("resource") DFSClient dfsClient = new DFSClient(NameNode.getAddress(getConf()), getConf()); DatanodeStorageReport[] datanodeStorageReports = dfsClient.getDatanodeStorageReport(DatanodeReportType.ALL); for (DatanodeStorageReport datanodeStorageReport : datanodeStorageReports) { disksPerHost.put( datanodeStorageReport.getDatanodeInfo().getHostName(), datanodeStorageReport.getStorageReports().length); } } catch (IOException e) { LOG.warn("number of data directories (disks) per node could not be collected (requieres higher privilegies)."); } return disksPerHost; }
@Override Double getUtilization(DatanodeStorageReport r, final StorageType t) { long capacity = 0L; long dfsUsed = 0L; for(StorageReport s : r.getStorageReports()) { if (s.getStorage().getStorageType() == t) { capacity += s.getCapacity(); dfsUsed += s.getDfsUsed(); } } return capacity == 0L? null: dfsUsed*100.0/capacity; } }
/** Get live datanode storage reports and then build the network topology. */ public List<DatanodeStorageReport> init() throws IOException { final DatanodeStorageReport[] reports = nnc.getLiveDatanodeStorageReport(); final List<DatanodeStorageReport> trimmed = new ArrayList<DatanodeStorageReport>(); // create network topology and classify utilization collections: // over-utilized, above-average, below-average and under-utilized. for (DatanodeStorageReport r : DFSUtil.shuffle(reports)) { final DatanodeInfo datanode = r.getDatanodeInfo(); if (shouldIgnore(datanode)) { continue; } trimmed.add(r); cluster.add(datanode); } return trimmed; }
public static DatanodeStorageReport convertDatanodeStorageReport( DatanodeStorageReportProto proto) { return new DatanodeStorageReport( convert(proto.getDatanodeInfo()), convertStorageReports(proto.getStorageReportsList())); }
public static DatanodeStorageReportProto convertDatanodeStorageReport( DatanodeStorageReport report) { return DatanodeStorageReportProto.newBuilder() .setDatanodeInfo(convert(report.getDatanodeInfo())) .addAllStorageReports(convertStorageReports(report.getStorageReports())) .build(); }
@Override Double getUtilization(DatanodeStorageReport r, final StorageType t) { long capacity = 0L; long blockPoolUsed = 0L; for(StorageReport s : r.getStorageReports()) { if (s.getStorage().getStorageType() == t) { capacity += s.getCapacity(); blockPoolUsed += s.getBlockPoolUsed(); } } return capacity == 0L? null: blockPoolUsed*100.0/capacity; } }
/** Get live datanode storage reports and then build the network topology. */ public List<DatanodeStorageReport> init() throws IOException { final DatanodeStorageReport[] reports = nnc.getLiveDatanodeStorageReport(); final List<DatanodeStorageReport> trimmed = new ArrayList<DatanodeStorageReport>(); // create network topology and classify utilization collections: // over-utilized, above-average, below-average and under-utilized. for (DatanodeStorageReport r : DFSUtil.shuffle(reports)) { final DatanodeInfo datanode = r.getDatanodeInfo(); if (shouldIgnore(datanode)) { continue; } trimmed.add(r); cluster.add(datanode); } return trimmed; }
public static DatanodeStorageReport convertDatanodeStorageReport( DatanodeStorageReportProto proto) { return new DatanodeStorageReport( convert(proto.getDatanodeInfo()), convertStorageReports(proto.getStorageReportsList())); }
public static DatanodeStorageReportProto convertDatanodeStorageReport( DatanodeStorageReport report) { return DatanodeStorageReportProto.newBuilder() .setDatanodeInfo(convert(report.getDatanodeInfo())) .addAllStorageReports(convertStorageReports(report.getStorageReports())) .build(); }
@Override void accumulateSpaces(DatanodeStorageReport r) { for(StorageReport s : r.getStorageReports()) { final StorageType t = s.getStorage().getStorageType(); totalCapacities.add(t, s.getCapacity()); totalUsedSpaces.add(t, s.getDfsUsed()); } }