LOG.debug("Found a {} Node {} {}", isAlive ? "living" : "dead", id, sup.getAllPorts()); nodeIdToNode.put(id, new Node(id, sup.getAllPorts(), isAlive)); if (node == null) { LOG.debug("Found an assigned slot on a dead supervisor {}", ws); node = new Node(id, null, false); nodeIdToNode.put(id, node); if (!node.isAlive()) { node.addOrphanedSlot(ws); if (node.assignInternal(ws, topId, true)) { LOG.warn("Bad scheduling state for topology [" + topId + "], the slot " + ws + " assigned to multiple workers, un-assigning everything..."); node.free(ws, cluster, true);
public static Collection<Node> takeNodesBySlot(int slotsNeeded, NodePool[] pools) { LOG.debug("Trying to grab {} free slots from {}", slotsNeeded, pools); HashSet<Node> ret = new HashSet<>(); for (NodePool pool : pools) { Collection<Node> got = pool.takeNodesBySlots(slotsNeeded); ret.addAll(got); slotsNeeded -= Node.countFreeSlotsAlive(got); LOG.debug("Got {} nodes so far need {} more slots", ret.size(), slotsNeeded); if (slotsNeeded <= 0) { break; } } return ret; }
@Override public Collection<Node> takeNodesBySlots(int slotsNeeded) { HashSet<Node> ret = new HashSet<>(); LinkedList<Node> sortedNodes = new LinkedList<>(_nodes); Collections.sort(sortedNodes, Node.FREE_NODE_COMPARATOR_DEC); for (Node n : sortedNodes) { if (slotsNeeded <= 0) { break; } if (n.isAlive()) { n.freeAllSlots(_cluster); _nodes.remove(n); ret.add(n); slotsNeeded -= n.totalSlotsFree(); } } return ret; }
@Override public Collection<Node> takeNodes(int nodesNeeded) { HashSet<Node> ret = new HashSet<>(); LinkedList<Node> sortedNodes = new LinkedList<>(_nodes); Collections.sort(sortedNodes, Node.FREE_NODE_COMPARATOR_DEC); for (Node n : sortedNodes) { if (nodesNeeded <= ret.size()) { break; } if (n.isAlive()) { n.freeAllSlots(_cluster); _nodes.remove(n); ret.add(n); } } return ret; }
@Override public Collection<Node> takeNodesBySlots(int slotsNeeded) { HashSet<Node> ret = new HashSet<>(); for (Entry<String, Set<Node>> entry : _topologyIdToNodes.entrySet()) { if (!_isolated.contains(entry.getKey())) { Iterator<Node> it = entry.getValue().iterator(); while (it.hasNext()) { Node n = it.next(); if (n.isAlive()) { it.remove(); _usedNodes--; n.freeAllSlots(_cluster); ret.add(n); slotsNeeded -= n.totalSlots(); if (slotsNeeded <= 0) { return ret; } } } } } return ret; }
int origRequest = td.getNumWorkers(); int slotsRequested = Math.min(totalTasks, origRequest); int slotsUsed = Node.countSlotsUsed(topId, _nodes); int slotsFree = Node.countFreeSlotsAlive(_nodes); n.freeTopology(topId, _cluster); slotsFree = Node.countFreeSlotsAlive(_nodes); slotsToUse = Math.min(slotsRequested, slotsFree); if (n.totalSlotsFree() == 0) { nodes.remove(); n = null; int afterSchedSlotsUsed = Node.countSlotsUsed(topId, _nodes); if (afterSchedSlotsUsed < slotsRequested) { _cluster.setStatus(topId, "Running with fewer slots than requested (" + afterSchedSlotsUsed + "/" + origRequest + ")");
int origRequest = td.getNumWorkers(); int slotsRequested = Math.min(totalTasks, origRequest); int slotsUsed = Node.countSlotsUsed(topId, allNodes); int slotsFree = Node.countFreeSlotsAlive(allNodes);
@Override public int slotsAvailable() { return Node.countTotalSlotsAlive(_nodes); }
@Override public Collection<Node> takeNodes(int nodesNeeded) { LOG.debug("Taking {} from {}", nodesNeeded, this); HashSet<Node> ret = new HashSet<>(); for (Entry<String, Set<Node>> entry : _topologyIdToNodes.entrySet()) { if (!_isolated.contains(entry.getKey())) { Iterator<Node> it = entry.getValue().iterator(); while (it.hasNext()) { if (nodesNeeded <= 0) { return ret; } Node n = it.next(); it.remove(); n.freeAllSlots(_cluster); ret.add(n); nodesNeeded--; _usedNodes--; } } } return ret; }
/** * Assign a free slot on the node to the following topology and executors. * This will update the cluster too. * @param topId the topology to assign a free slot to. * @param executors the executors to run in that slot. * @param cluster the cluster to be updated */ public void assign(String topId, Collection<ExecutorDetails> executors, Cluster cluster) { if (!_isAlive) { throw new IllegalStateException("Trying to adding to a dead node " + _nodeId); } if (_freeSlots.isEmpty()) { throw new IllegalStateException("Trying to assign to a full node " + _nodeId); } if (executors.size() == 0) { LOG.warn("Trying to assign nothing from " + topId + " to " + _nodeId + " (Ignored)"); } else { WorkerSlot slot = _freeSlots.iterator().next(); cluster.assign(slot, topId, executors); assignInternal(slot, topId, false); } }
int origRequest = td.getNumWorkers(); int slotsRequested = Math.min(totalTasks, origRequest); int slotsUsed = Node.countSlotsUsed(topId, _nodes); int slotsFree = Node.countFreeSlotsAlive(_nodes); n.freeTopology(topId, _cluster); slotsFree = Node.countFreeSlotsAlive(_nodes); slotsToUse = Math.min(slotsRequested, slotsFree); if (n.totalSlotsFree() == 0) { nodes.remove(); n = null; int afterSchedSlotsUsed = Node.countSlotsUsed(topId, _nodes); if (afterSchedSlotsUsed < slotsRequested) { _cluster.setStatus(topId,"Running with fewer slots than requested ("+afterSchedSlotsUsed+"/"+origRequest+")");
@Override public Collection<Node> takeNodesBySlots(int slotsNeeded) { HashSet<Node> ret = new HashSet<>(); LinkedList<Node> sortedNodes = new LinkedList<>(_nodes); Collections.sort(sortedNodes, Node.FREE_NODE_COMPARATOR_DEC); for (Node n: sortedNodes) { if (slotsNeeded <= 0) { break; } if (n.isAlive()) { n.freeAllSlots(_cluster); _nodes.remove(n); ret.add(n); slotsNeeded -= n.totalSlotsFree(); } } return ret; }
@Override public Collection<Node> takeNodesBySlots(int slotsNeeded) { HashSet<Node> ret = new HashSet<>(); for (Entry<String, Set<Node>> entry: _topologyIdToNodes.entrySet()) { if (!_isolated.contains(entry.getKey())) { Iterator<Node> it = entry.getValue().iterator(); while (it.hasNext()) { Node n = it.next(); if (n.isAlive()) { it.remove(); _usedNodes--; n.freeAllSlots(_cluster); ret.add(n); slotsNeeded -= n.totalSlots(); if (slotsNeeded <= 0) { return ret; } } } } } return ret; }
int origRequest = td.getNumWorkers(); int slotsRequested = Math.min(totalTasks, origRequest); int slotsUsed = Node.countSlotsUsed(allNodes); int slotsFree = Node.countFreeSlotsAlive(allNodes); int slotsToUse = Math.min(slotsRequested - slotsUsed, slotsFree); if (slotsToUse <= 0) {
@Override public Collection<Node> takeNodes(int nodesNeeded) { HashSet<Node> ret = new HashSet<>(); LinkedList<Node> sortedNodes = new LinkedList<>(_nodes); Collections.sort(sortedNodes, Node.FREE_NODE_COMPARATOR_DEC); for (Node n: sortedNodes) { if (nodesNeeded <= ret.size()) { break; } if (n.isAlive()) { n.freeAllSlots(_cluster); _nodes.remove(n); ret.add(n); } } return ret; }
@Override public int slotsAvailable() { int total = 0; for (Entry<String, Set<Node>> entry : _topologyIdToNodes.entrySet()) { if (!_isolated.contains(entry.getKey())) { total += Node.countTotalSlotsAlive(entry.getValue()); } } return total; }
@Override public Collection<Node> takeNodes(int nodesNeeded) { LOG.debug("Taking {} from {}", nodesNeeded, this); HashSet<Node> ret = new HashSet<>(); for (Entry<String, Set<Node>> entry: _topologyIdToNodes.entrySet()) { if (!_isolated.contains(entry.getKey())) { Iterator<Node> it = entry.getValue().iterator(); while (it.hasNext()) { if (nodesNeeded <= 0) { return ret; } Node n = it.next(); it.remove(); n.freeAllSlots(_cluster); ret.add(n); nodesNeeded--; _usedNodes--; } } } return ret; }
/** * Assign a free slot on the node to the following topology and executors. * This will update the cluster too. * @param topId the topology to assign a free slot to. * @param executors the executors to run in that slot. * @param cluster the cluster to be updated */ public void assign(String topId, Collection<ExecutorDetails> executors, Cluster cluster) { if (!_isAlive) { throw new IllegalStateException("Trying to adding to a dead node " + _nodeId); } if (_freeSlots.isEmpty()) { throw new IllegalStateException("Trying to assign to a full node " + _nodeId); } if (executors.size() == 0) { LOG.warn("Trying to assign nothing from " + topId + " to " + _nodeId + " (Ignored)"); } else { WorkerSlot slot = _freeSlots.iterator().next(); cluster.assign(slot, topId, executors); assignInternal(slot, topId, false); } }
LOG.debug("Found a {} Node {} {}", isAlive? "living":"dead", id, sup.getAllPorts()); nodeIdToNode.put(id, new Node(id, sup.getAllPorts(), isAlive)); if (node == null) { LOG.debug("Found an assigned slot on a dead supervisor {}", ws); node = new Node(id, null, false); nodeIdToNode.put(id, node); if (!node.isAlive()) { node.addOrphanedSlot(ws); if (node.assignInternal(ws, topId, true)) { LOG.warn("Bad scheduling state for topology [" + topId+ "], the slot " + ws + " assigned to multiple workers, un-assigning everything..."); node.free(ws, cluster, true);