private ClusterResourceDescription getCurrentFreeClusterResources(YarnClient yarnClient) throws YarnException, IOException { List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING); int totalFreeMemory = 0; int containerLimit = 0; int[] nodeManagersFree = new int[nodes.size()]; for (int i = 0; i < nodes.size(); i++) { NodeReport rep = nodes.get(i); int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep.getUsed().getMemory() : 0); nodeManagersFree[i] = free; totalFreeMemory += free; if (free > containerLimit) { containerLimit = free; } } return new ClusterResourceDescription(totalFreeMemory, containerLimit, nodeManagersFree); }
private ClusterResourceDescription getCurrentFreeClusterResources(YarnClient yarnClient) throws YarnException, IOException { List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING); int totalFreeMemory = 0; int containerLimit = 0; int[] nodeManagersFree = new int[nodes.size()]; for (int i = 0; i < nodes.size(); i++) { NodeReport rep = nodes.get(i); int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep.getUsed().getMemory() : 0); nodeManagersFree[i] = free; totalFreeMemory += free; if (free > containerLimit) { containerLimit = free; } } return new ClusterResourceDescription(totalFreeMemory, containerLimit, nodeManagersFree); }
private ClusterResourceDescription getCurrentFreeClusterResources(YarnClient yarnClient) throws YarnException, IOException { List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING); int totalFreeMemory = 0; int containerLimit = 0; int[] nodeManagersFree = new int[nodes.size()]; for (int i = 0; i < nodes.size(); i++) { NodeReport rep = nodes.get(i); int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep.getUsed().getMemory() : 0); nodeManagersFree[i] = free; totalFreeMemory += free; if (free > containerLimit) { containerLimit = free; } } return new ClusterResourceDescription(totalFreeMemory, containerLimit, nodeManagersFree); }
private ClusterResourceDescription getCurrentFreeClusterResources( YarnClient yarnClient) throws YarnException, IOException { List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING); int totalFreeMemory = 0; int containerLimit = 0; int[] nodeManagersFree = new int[nodes.size()]; for (int i = 0; i < nodes.size(); i++) { NodeReport rep = nodes.get(i); int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep. getUsed().getMemory() : 0); nodeManagersFree[i] = free; totalFreeMemory += free; if (free > containerLimit) { containerLimit = free; } } return new ClusterResourceDescription(totalFreeMemory, containerLimit, nodeManagersFree); }
/** * Assign host to container given affinity and anti-affinity constraints and resource availibility on node * @param host * @param antiHosts * @param antiPreferredHosts * @param grpObj * @param nodeLocalSet * @param aggrMemory * @param vCores * @return */ public String assignHost(String host, List<String> antiHosts, List<String> antiPreferredHosts, HostOperatorSet grpObj, Set<PTOperator> nodeLocalSet, int aggrMemory, int vCores) { for (Map.Entry<String, NodeReport> nodeEntry : nodeReportMap.entrySet()) { if (nodeEntry.getValue().getNodeState() == NodeState.RUNNING) { int memAvailable = nodeEntry.getValue().getCapability().getMemory() - nodeEntry.getValue().getUsed().getMemory(); int vCoresAvailable = nodeEntry.getValue().getCapability().getVirtualCores() - nodeEntry.getValue().getUsed().getVirtualCores(); if (memAvailable >= aggrMemory && vCoresAvailable >= vCores && !antiHosts.contains(nodeEntry.getKey()) && !antiPreferredHosts.contains(nodeEntry.getKey())) { host = nodeEntry.getKey(); grpObj.setHost(host); nodeLocalMapping.put(nodeLocalSet, host); return host; } } } return null; }
nodeReportStr.println(nodeReport.getNumContainers()); nodeReportStr.print("\tMemory-Used : "); nodeReportStr.println((nodeReport.getUsed() == null) ? "0MB" : (nodeReport.getUsed().getMemory() + "MB")); nodeReportStr.print("\tMemory-Capacity : "); nodeReportStr.println(nodeReport.getCapability().getMemory() + "MB"); nodeReportStr.print("\tCPU-Used : "); nodeReportStr.println((nodeReport.getUsed() == null) ? "0 vcores" : (nodeReport.getUsed().getVirtualCores() + " vcores")); nodeReportStr.print("\tCPU-Capacity : "); nodeReportStr.println(nodeReport.getCapability().getVirtualCores() + " vcores");
nodeReportStr.println(nodeReport.getNumContainers()); nodeReportStr.print("\tMemory-Used : "); nodeReportStr.println((nodeReport.getUsed() == null) ? "0MB" : (nodeReport.getUsed().getMemory() + "MB")); nodeReportStr.print("\tMemory-Capacity : "); nodeReportStr.println(nodeReport.getCapability().getMemory() + "MB"); nodeReportStr.print("\tCPU-Used : "); nodeReportStr.println((nodeReport.getUsed() == null) ? "0 vcores" : (nodeReport.getUsed().getVirtualCores() + " vcores")); nodeReportStr.print("\tCPU-Capacity : "); nodeReportStr.println(nodeReport.getCapability().getVirtualCores() + " vcores");
nodeReportStr.println(nodeReport.getNumContainers()); nodeReportStr.print("\tMemory-Used : "); nodeReportStr.println((nodeReport.getUsed() == null) ? "0MB" : (nodeReport.getUsed().getMemorySize() + "MB")); nodeReportStr.print("\tMemory-Capacity : "); nodeReportStr.println(nodeReport.getCapability().getMemorySize() + "MB"); nodeReportStr.print("\tCPU-Used : "); nodeReportStr.println((nodeReport.getUsed() == null) ? "0 vcores" : (nodeReport.getUsed().getVirtualCores() + " vcores")); nodeReportStr.print("\tCPU-Capacity : "); nodeReportStr.println(nodeReport.getCapability().getVirtualCores() + " vcores"); nodeReportStr.print("\tGPU-Used : "); nodeReportStr.println((nodeReport.getUsed() == null) ? "0 gpus" : (nodeReport.getUsed().getGPUs() + " gpus")); nodeReportStr.print("\tGPU-Capacity : "); nodeReportStr.println(nodeReport.getCapability().getGPUs() + " " +
nodeReportStr.println(nodeReport.getNumContainers()); nodeReportStr.print("\tMemory-Used : "); nodeReportStr.println((nodeReport.getUsed() == null) ? "0MB" : (nodeReport.getUsed().getMemorySize() + "MB")); nodeReportStr.print("\tMemory-Capacity : "); nodeReportStr.println(nodeReport.getCapability().getMemorySize() + "MB"); nodeReportStr.print("\tCPU-Used : "); nodeReportStr.println((nodeReport.getUsed() == null) ? "0 vcores" : (nodeReport.getUsed().getVirtualCores() + " vcores")); nodeReportStr.print("\tCPU-Capacity : "); nodeReportStr.println(nodeReport.getCapability().getVirtualCores() + " vcores");
/** * Tracks update to available resources. Resource availability is used to make decisions about where to request new * containers. * * @param nodeReports */ public void updateNodeReports(List<NodeReport> nodeReports) { for (NodeReport nr : nodeReports) { LOG.debug("Node report: rackName={}, nodeid={}, numContainers={}, capability={}, used={}, state={}", nr.getRackName(), nr.getNodeId(), nr.getNumContainers(), nr.getCapability(), nr.getUsed(), nr.getNodeState()); nodeReportMap.put(nr.getNodeId().getHost(), nr); nodeToRack.put(nr.getNodeId().getHost(), nr.getRackName()); } }
/** * Helper function to verify DISTRIBUTED placement policies. * Returns the number of NodeManagers on which runnables got provisioned. * @return number of NodeManagers on which runnables got provisioned. */ private int getProvisionedNodeManagerCount() throws Exception { int provisionedNodeManagerCount = 0; for (NodeReport nodeReport : getNodeReports()) { Resource used = nodeReport.getUsed(); if (used != null && used.getMemory() > 0) { provisionedNodeManagerCount++; } } return provisionedNodeManagerCount; } }
if (!nodeReport.getNodeState().isUnusable()) { Resource nodeCapability = nodeReport.getCapability(); Resource nodeUsed = nodeReport.getUsed();
private void reportYarnResources() { try { long totalMemory = 0L; long totalVCores = 0L; long usedMemory = 0L; long usedVCores = 0L; for (NodeReport nodeReport : yarnClient.getNodeReports(NodeState.RUNNING)) { Resource capability = nodeReport.getCapability(); Resource used = nodeReport.getUsed(); totalMemory += capability.getMemory(); totalVCores += YarnUtils.getVirtualCores(capability); usedMemory += used.getMemory(); usedVCores += YarnUtils.getVirtualCores(used); } MetricsContext collector = getCollector(); LOG.trace("YARN Cluster memory total={}MB, used={}MB", totalMemory, usedMemory); collector.gauge("resources.total.memory", totalMemory); collector.gauge("resources.used.memory", usedMemory); collector.gauge("resources.available.memory", totalMemory - usedMemory); LOG.trace("YARN Cluster vcores total={}, used={}", totalVCores, usedVCores); collector.gauge("resources.total.vcores", totalVCores); collector.gauge("resources.used.vcores", usedVCores); collector.gauge("resources.available.vcores", totalVCores - usedVCores); } catch (Exception e) { LOG.warn("Failed to gather YARN NodeReports", e); } }
private void reportYarnResources() { try { long totalMemory = 0L; long totalVCores = 0L; long usedMemory = 0L; long usedVCores = 0L; for (NodeReport nodeReport : yarnClient.getNodeReports(NodeState.RUNNING)) { Resource capability = nodeReport.getCapability(); Resource used = nodeReport.getUsed(); totalMemory += capability.getMemory(); totalVCores += YarnUtils.getVirtualCores(capability); usedMemory += used.getMemory(); usedVCores += YarnUtils.getVirtualCores(used); } MetricsContext collector = getCollector(); LOG.trace("YARN Cluster memory total={}MB, used={}MB", totalMemory, usedMemory); collector.gauge("resources.total.memory", totalMemory); collector.gauge("resources.used.memory", usedMemory); collector.gauge("resources.available.memory", totalMemory - usedMemory); LOG.trace("YARN Cluster vcores total={}, used={}", totalVCores, usedVCores); collector.gauge("resources.total.vcores", totalVCores); collector.gauge("resources.used.vcores", usedVCores); collector.gauge("resources.available.vcores", totalVCores - usedVCores); } catch (Exception e) { LOG.warn("Failed to gather YARN NodeReports", e); } }
Resource used = nodeReport.getUsed(); Assert.assertNotNull(capability); if (used != null) {