@Override public String toString() { return getId(); } }
/** * Frees a single slot in this node. * * @param ws the slot to free */ public void free(WorkerSlot ws) { LOG.debug("freeing WorkerSlot {} on node {}", ws, hostname); if (!slots.containsKey(ws.getId())) { throw new IllegalArgumentException( "Tried to free a slot " + ws + " that was not" + " part of this node " + nodeId); } TopologyDetails topo = findTopologyUsingWorker(ws); if (topo == null) { throw new IllegalArgumentException("Tried to free a slot " + ws + " that was already free!"); } //free slot cluster.freeSlot(ws); //cleanup internal assignments topIdToUsedSlots.get(topo.getId()).remove(ws.getId()); }
/** * Find a which topology is running on a worker slot. * * @return the topology using the worker slot. If worker slot is free then return null */ private TopologyDetails findTopologyUsingWorker(WorkerSlot ws) { for (Entry<String, Map<String, Collection<ExecutorDetails>>> entry : topIdToUsedSlots.entrySet()) { String topoId = entry.getKey(); Set<String> workerIds = entry.getValue().keySet(); for (String workerId : workerIds) { if (ws.getId().equals(workerId)) { return cluster.getTopologies().getById(topoId); } } } return null; }
if (freeById.contains(slot.getId())) { originallyFreeSlots.add(slot);
/** * Assign a single executor to a slot, even if other things are in the slot. * @param ws the slot to assign it to. * @param exec the executor to assign. * @param td the topology for the executor. */ public void assignSingleExecutor(WorkerSlot ws, ExecutorDetails exec, TopologyDetails td) { if (!isAlive) { throw new IllegalStateException("Trying to adding to a dead node " + nodeId); } Collection<WorkerSlot> freeSlots = getFreeSlots(); Set<ExecutorDetails> toAssign = new HashSet<>(); toAssign.add(exec); if (!freeSlots.contains(ws)) { Map<String, Collection<ExecutorDetails>> usedSlots = topIdToUsedSlots.get(td.getId()); if (usedSlots == null) { throw new IllegalArgumentException( "Slot " + ws + " is not availble to schedue " + exec + " on"); } Collection<ExecutorDetails> alreadyHere = usedSlots.get(ws.getId()); if (alreadyHere == null) { throw new IllegalArgumentException( "Slot " + ws + " is not availble to schedue " + exec + " on"); } toAssign.addAll(alreadyHere); free(ws); } assign(ws, td, toAssign); }
.computeIfAbsent(target.getId(), (tid) -> new LinkedList<>()) .addAll(executors);
workerIdToWorker.put(nodeId, new HashMap<String, WorkerSlot>()); workerIdToWorker.get(nodeId).put(slot.getId(), slot); if (!assignmentRelationshipMap.get(nodeId).containsKey(topId)) { assignmentRelationshipMap .put(topId, new HashMap<String, Collection<ExecutorDetails>>()); if (!assignmentRelationshipMap.get(nodeId).get(topId).containsKey(slot.getId())) { assignmentRelationshipMap .get(nodeId) .get(topId) .put(slot.getId(), new LinkedList<ExecutorDetails>()); assignmentRelationshipMap.get(nodeId).get(topId).get(slot.getId()).addAll(execs); workerIdToWorker.put(sup.getId(), new HashMap<String, WorkerSlot>()); if (!workerIdToWorker.get(sup.getId()).containsKey(worker.getId())) { workerIdToWorker.get(sup.getId()).put(worker.getId(), worker);
/** * Find a which topology is running on a worker slot * @return the topology using the worker slot. If worker slot is free then return null */ public TopologyDetails findTopologyUsingWorker(WorkerSlot ws) { for (Entry<String, Map<String, Collection<ExecutorDetails>>> entry : _topIdToUsedSlots.entrySet()) { String topoId = entry.getKey(); Set<String> workerIds = entry.getValue().keySet(); for (String workerId : workerIds) { if(ws.getId().equals(workerId)) { return _topologies.getById(topoId); } } } return null; }
/** * Frees a single slot in this node * @param ws the slot to free */ public void free(WorkerSlot ws) { LOG.info("freeing WorkerSlot {} on node {}", ws, _hostname); if (!_slots.containsKey(ws.getId())) { throw new IllegalArgumentException("Tried to free a slot " + ws + " that was not" + " part of this node " + _nodeId); } TopologyDetails topo = findTopologyUsingWorker(ws); if (topo == null) { throw new IllegalArgumentException("Tried to free a slot " + ws + " that was already free!"); } double memUsed = getMemoryUsedByWorker(ws); double cpuUsed = getCpuUsedByWorker(ws); freeMemory(memUsed); freeCPU(cpuUsed); //free slot _cluster.freeSlot(ws); //cleanup internal assignments _topIdToUsedSlots.get(topo.getId()).remove(ws.getId()); }
workerIdToWorker.put(nodeId, new HashMap<String, WorkerSlot>()); workerIdToWorker.get(nodeId).put(slot.getId(), slot); if (!assignmentRelationshipMap.get(nodeId).containsKey(topId)) { assignmentRelationshipMap.get(nodeId).put(topId, new HashMap<String, Collection<ExecutorDetails>>()); if (!assignmentRelationshipMap.get(nodeId).get(topId).containsKey(slot.getId())) { assignmentRelationshipMap.get(nodeId).get(topId).put(slot.getId(), new LinkedList<ExecutorDetails>()); assignmentRelationshipMap.get(nodeId).get(topId).get(slot.getId()).addAll(execs); workerIdToWorker.put(sup.getId(), new HashMap<String, WorkerSlot>()); if (!workerIdToWorker.get(sup.getId()).containsKey(worker.getId())) { workerIdToWorker.get(sup.getId()).put(worker.getId(), worker);