/** {@inheritDoc} */ public synchronized void onNodesUpdated(List<NodeReport> updated) { for (NodeReport node : updated) { // If node unusable. if (node.getNodeState().isUnusable()) { for (IgniteContainer cont : containers.values()) { if (cont.nodeId().equals(node.getNodeId())) { containers.remove(cont.id()); log.log(Level.WARNING, "Node is unusable. Node: {0}, state: {1}.", new Object[]{node.getNodeId().getHost(), node.getNodeState()}); } } log.log(Level.WARNING, "Node is unusable. Node: {0}, state: {1}.", new Object[]{node.getNodeId().getHost(), node.getNodeState()}); } } }
int totalCores = 0; for (NodeReport rep : nodes) { final Resource res = rep.getCapability(); totalMemory += res.getMemory(); totalCores += res.getVirtualCores(); ps.format(format, "NodeID", rep.getNodeId()); ps.format(format, "Memory", res.getMemory() + " MB"); ps.format(format, "vCores", res.getVirtualCores()); ps.format(format, "HealthReport", rep.getHealthReport()); ps.format(format, "Containers", rep.getNumContainers()); ps.println("+---------------------------------------+");
private void buildNodeMap() throws YarnFacadeException { List<NodeReport> nodes = yarn.getNodeReports(); for (NodeReport node : nodes) { String hostName = node.getNodeId().getHost(); nodeMap.put(hostName, node.getHttpAddress()); yarnNodes.put(hostName, node); } if (LOG.isInfoEnabled()) { LOG.info("YARN Node report"); for (NodeReport node : nodes) { LOG.info("Node: " + node.getHttpAddress() + ", Rack: " + node.getRackName() + " has " + node.getCapability().getMemory() + " MB, " + node.getCapability().getVirtualCores() + " vcores, labels: " + node.getNodeLabels()); } } }
private ClusterResourceDescription getCurrentFreeClusterResources(YarnClient yarnClient) throws YarnException, IOException { List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING); int totalFreeMemory = 0; int containerLimit = 0; int[] nodeManagersFree = new int[nodes.size()]; for (int i = 0; i < nodes.size(); i++) { NodeReport rep = nodes.get(i); int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep.getUsed().getMemory() : 0); nodeManagersFree[i] = free; totalFreeMemory += free; if (free > containerLimit) { containerLimit = free; } } return new ClusterResourceDescription(totalFreeMemory, containerLimit, nodeManagersFree); }
NodeReport nodeReport = null; for (NodeReport report : nodesReport) { if (!report.getNodeId().equals(nodeId)) { continue; nodeReportStr.println("Node Report : "); nodeReportStr.print("\tNode-Id : "); nodeReportStr.println(nodeReport.getNodeId()); nodeReportStr.print("\tRack : "); nodeReportStr.println(nodeReport.getRackName()); nodeReportStr.print("\tNode-State : "); nodeReportStr.println(nodeReport.getNodeState()); nodeReportStr.print("\tNode-Http-Address : "); nodeReportStr.println(nodeReport.getHttpAddress()); nodeReportStr.print("\tLast-Health-Update : "); nodeReportStr.println(DateFormatUtils.format( new Date(nodeReport.getLastHealthReportTime()), "E dd/MMM/yy hh:mm:ss:SSzz")); nodeReportStr.print("\tHealth-Report : "); nodeReportStr .println(nodeReport.getHealthReport()); nodeReportStr.print("\tContainers : "); nodeReportStr.println(nodeReport.getNumContainers()); nodeReportStr.print("\tMemory-Used : "); nodeReportStr.println((nodeReport.getUsed() == null) ? "0MB" : (nodeReport.getUsed().getMemory() + "MB")); nodeReportStr.print("\tMemory-Capacity : "); nodeReportStr.println(nodeReport.getCapability().getMemory() + "MB"); nodeReportStr.print("\tCPU-Used : ");
"Number-of-Running-Containers"); for (NodeReport nodeReport : nodesReport) { writer.printf(NODES_PATTERN, nodeReport.getNodeId(), nodeReport.getNodeState(), nodeReport.getHttpAddress(), nodeReport.getNumContainers()); writer.println("Detailed Node Information :"); writer.print("\tConfigured Resources : "); writer.println(nodeReport.getCapability()); writer.print("\tAllocated Resources : "); if (nodeReport.getUsed() != null) { writer.print(nodeReport.getUsed()); if (nodeReport.getNodeUtilization() != null) { writer.print("PMem:" + nodeReport.getNodeUtilization().getPhysicalMemory() + " MB, VMem:" + nodeReport.getNodeUtilization().getVirtualMemory() + " MB, VCores:" + nodeReport.getNodeUtilization().getCPU()); if (nodeReport.getAggregatedContainersUtilization() != null) { writer.print("PMem:" + nodeReport.getAggregatedContainersUtilization() .getPhysicalMemory() + " MB, VMem:" + nodeReport.getAggregatedContainersUtilization() .getVirtualMemory() + " MB, VCores:" + nodeReport.getAggregatedContainersUtilization().getCPU()); nodeReport.getNodeLabels()); Collections.sort(nodeLabelsList);
/** * Lists the nodes matching the given node states * * @param nodeStates * @throws YarnException * @throws IOException */ private void listClusterNodes(Set<NodeState> nodeStates) throws YarnException, IOException { PrintWriter writer = new PrintWriter( new OutputStreamWriter(sysout, Charset.forName("UTF-8"))); List<NodeReport> nodesReport = client.getNodeReports( nodeStates.toArray(new NodeState[0])); writer.println("Total Nodes:" + nodesReport.size()); writer.printf(NODES_PATTERN, "Node-Id", "Node-State", "Node-Http-Address", "Number-of-Running-Containers"); for (NodeReport nodeReport : nodesReport) { writer.printf(NODES_PATTERN, nodeReport.getNodeId(), nodeReport .getNodeState(), nodeReport.getHttpAddress(), nodeReport .getNumContainers()); } writer.flush(); }
NodeInformationList results = new NodeInformationList(reports.size()); for (NodeReport report : reports) { if (live && report.getNodeState() != NodeState.RUNNING) { continue; if (!label.isEmpty() && !report.getNodeLabels().contains(label)) { continue; info.hostname = report.getNodeId().getHost(); info.healthReport = report.getHealthReport(); info.httpAddress = report.getHttpAddress(); info.labels = SliderUtils.extractNodeLabel(report); info.rackName = report.getRackName(); info.state = report.getNodeState().toString(); results.add(info);
for (NodeReport node : clusterNodeReports) { LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers());
/** * Tracks update to available resources. Resource availability is used to make decisions about where to request new * containers. * * @param nodeReports */ public void updateNodeReports(List<NodeReport> nodeReports) { for (NodeReport nr : nodeReports) { LOG.debug("Node report: rackName={}, nodeid={}, numContainers={}, capability={}, used={}, state={}", nr.getRackName(), nr.getNodeId(), nr.getNumContainers(), nr.getCapability(), nr.getUsed(), nr.getNodeState()); nodeReportMap.put(nr.getNodeId().getHost(), nr); nodeToRack.put(nr.getNodeId().getHost(), nr.getRackName()); } }
@Override public void onNodesUpdated(List<NodeReport> nodeReports) { LOG.debug("Received nodes update for '{}' nodes", nodeReports.size()); for (NodeReport nodeReport : nodeReports) { if (nodeReport.getNodeState() == NodeState.RUNNING) { String nodeKey = getNodeName(nodeReport.getNodeId()); nodes.put(nodeKey, nodeReport.getCapability()); LOG.debug("Added node '{}' with '{}' cpus and '{}' memory", nodeKey, nodeReport.getCapability().getVirtualCores(), nodeReport.getCapability().getMemory()); } else { LOG.debug("Removed node '{}'", nodeReport.getNodeId()); nodes.remove(getNodeName(nodeReport.getNodeId())); } } }
private void onNodeReport(final NodeReport nodeReport) { LOG.log(Level.FINE, "Send node descriptor: {0}", nodeReport); this.reefEventHandlers.onNodeDescriptor(NodeDescriptorEventImpl.newBuilder() .setIdentifier(nodeReport.getNodeId().toString()) .setHostName(nodeReport.getNodeId().getHost()) .setPort(nodeReport.getNodeId().getPort()) .setMemorySize(nodeReport.getCapability().getMemory()) .setRackName(nodeReport.getRackName()) .build()); }
public static TaskTrackerInfo fromYarn(NodeReport node) { TaskTrackerInfo taskTracker = new TaskTrackerInfo(TT_NAME_PREFIX + node.getNodeId().toString()); return taskTracker; }
int availableNodes = 0; for (NodeReport nodeReport : nodeReports) { NodeId nodeId = nodeReport.getNodeId(); LOG.debug("Got report for node {}", nodeId); if (!nodeReport.getNodeState().isUnusable()) { Resource nodeCapability = nodeReport.getCapability(); Resource nodeUsed = nodeReport.getUsed();
Assert.assertEquals(1, updatedNodes.size()); NodeReport nr = updatedNodes.iterator().next(); Assert.assertEquals(nm4.getNodeId(), nr.getNodeId()); Assert.assertEquals(NodeState.UNHEALTHY, nr.getNodeState()); Assert.assertNull(nr.getDecommissioningTimeout()); Assert.assertEquals(NodeUpdateType.NODE_UNUSABLE, nr.getNodeUpdateType()); Assert.assertEquals(1, updatedNodes.size()); nr = updatedNodes.iterator().next(); Assert.assertEquals(nm4.getNodeId(), nr.getNodeId()); Assert.assertEquals(NodeState.UNHEALTHY, nr.getNodeState()); Assert.assertNull(nr.getDecommissioningTimeout()); Assert.assertEquals(NodeUpdateType.NODE_UNUSABLE, nr.getNodeUpdateType()); Assert.assertEquals(1, updatedNodes.size()); nr = updatedNodes.iterator().next(); Assert.assertEquals(nm3.getNodeId(), nr.getNodeId()); Assert.assertEquals(NodeState.LOST, nr.getNodeState()); Assert.assertNull(nr.getDecommissioningTimeout()); Assert.assertEquals(NodeUpdateType.NODE_UNUSABLE, nr.getNodeUpdateType()); Assert.assertEquals(1, updatedNodes.size()); nr = updatedNodes.iterator().next(); Assert.assertEquals(nm4.getNodeId(), nr.getNodeId()); Assert.assertEquals(NodeState.RUNNING, nr.getNodeState()); Assert.assertNull(nr.getDecommissioningTimeout()); Assert.assertEquals(NodeUpdateType.NODE_USABLE, nr.getNodeUpdateType()); Assert.assertEquals(1, updatedNodes.size());
@Override public List<NodeInfo> getNodes() throws LlamaException { List<NodeInfo> ret = new ArrayList<NodeInfo>(); try { if (nodes == null) { // Get it from the yarn client. List<NodeReport> nodeReports = yarnClient.getNodeReports(NodeState.RUNNING); for (NodeReport nodeReport : nodeReports) { Resource resource = nodeReport.getCapability(); NodeInfo nodeInfo = new NodeInfo(getNodeName(nodeReport.getNodeId()), resource.getVirtualCores(), resource.getMemory()); ret.add(nodeInfo); } } else { // Get it from the nodes structure which is being kept upto date. synchronized (nodes) { for(Map.Entry<String, Resource> entry : nodes.entrySet()) { Resource nodeReport = entry.getValue(); NodeInfo nodeInfo = new NodeInfo(entry.getKey(), nodeReport.getVirtualCores(), nodeReport.getMemory()); ret.add(nodeInfo); } } } return ret; } catch (Throwable ex) { throw new LlamaException(ex, ErrorCode.AM_CANNOT_GET_NODES, appId); } }
/** * Assign host to container given affinity and anti-affinity constraints and resource availibility on node * @param host * @param antiHosts * @param antiPreferredHosts * @param grpObj * @param nodeLocalSet * @param aggrMemory * @param vCores * @return */ public String assignHost(String host, List<String> antiHosts, List<String> antiPreferredHosts, HostOperatorSet grpObj, Set<PTOperator> nodeLocalSet, int aggrMemory, int vCores) { for (Map.Entry<String, NodeReport> nodeEntry : nodeReportMap.entrySet()) { if (nodeEntry.getValue().getNodeState() == NodeState.RUNNING) { int memAvailable = nodeEntry.getValue().getCapability().getMemory() - nodeEntry.getValue().getUsed().getMemory(); int vCoresAvailable = nodeEntry.getValue().getCapability().getVirtualCores() - nodeEntry.getValue().getUsed().getVirtualCores(); if (memAvailable >= aggrMemory && vCoresAvailable >= vCores && !antiHosts.contains(nodeEntry.getKey()) && !antiPreferredHosts.contains(nodeEntry.getKey())) { host = nodeEntry.getKey(); grpObj.setHost(host); nodeLocalMapping.put(nodeLocalSet, host); return host; } } } return null; }
/** * Produced a serialized form which can be served up as JSON * @param naming map of priority -> value for naming entries * @return a summary of the current role status. */ public synchronized NodeInformation serialize(Map<Integer, String> naming) { NodeInformation info = new NodeInformation(); info.hostname = hostname; // null-handling state constructor info.state = "" + nodeState; info.lastUpdated = nodeStateUpdateTime; info.labels = nodeLabels; if (nodeReport != null) { info.httpAddress = nodeReport.getHttpAddress(); info.rackName = nodeReport.getRackName(); info.healthReport = nodeReport.getHealthReport(); } info.entries = new HashMap<>(nodeEntries.size()); for (NodeEntry nodeEntry : nodeEntries) { String name = naming.get(nodeEntry.rolePriority); if (name == null) { name = Integer.toString(nodeEntry.rolePriority); } info.entries.put(name, nodeEntry.serialize()); } return info; }
/** * RM Callback: RM reports status change of some nodes. * @param nodeReports list of nodes with changed status. */ @Override public void onNodesUpdated(final List<NodeReport> nodeReports) { for (final NodeReport nodeReport : nodeReports) { this.nodeIdToRackName.put(nodeReport.getNodeId().toString(), nodeReport.getRackName()); this.onNodeReport(nodeReport); } }
Assert.assertEquals(1, nodeReports.size()); Assert.assertNotSame("Node is expected to be healthy!", NodeState.UNHEALTHY, nodeReports.get(0).getNodeState()); Assert.assertEquals(1, nodeReports.size()); Assert.assertEquals("G", nodeReports.get(0).getCapability(). getResourceInformation("resource1").getUnits()); Assert.assertEquals(1000, nodeReports.get(0).getCapability(). getResourceInformation("resource1").getValue()); Assert.assertEquals("m", nodeReports.get(0).getCapability(). getResourceInformation("resource2").getUnits()); Assert.assertEquals(1000000000, nodeReports.get(0).getCapability(). getResourceInformation("resource2").getValue()); Assert.assertEquals("Mi", nodeReports.get(0).getCapability(). getResourceInformation("memory-mb").getUnits()); Assert.assertEquals(976562, nodeReports.get(0).getCapability(). getResourceInformation("memory-mb").getValue());