/** * Get all topologies submitted/owned by a given user. * * @param user the name of the user * @return all of the topologies submitted by this user. */ public Collection<TopologyDetails> getTopologiesOwnedBy(String user) { HashSet<TopologyDetails> ret = new HashSet<>(); for (TopologyDetails td : this) { if (user.equals(td.getTopologySubmitter())) { ret.add(td); } } return ret; }
/** * Get User wrappers around cluster. * * @param cluster the cluster to get the users out of. */ private Map<String, User> getUsers(Cluster cluster) { Map<String, User> userMap = new HashMap<>(); Map<String, Map<String, Double>> userResourcePools = getUserResourcePools(); LOG.debug("userResourcePools: {}", userResourcePools); for (TopologyDetails td : cluster.getTopologies()) { String topologySubmitter = td.getTopologySubmitter(); //additional safety check to make sure that topologySubmitter is going to be a valid value if (topologySubmitter == null || topologySubmitter.equals("")) { LOG.error("Cannot determine user for topology {}. Will skip scheduling this topology", td.getName()); continue; } if (!userMap.containsKey(topologySubmitter)) { userMap.put(topologySubmitter, new User(topologySubmitter, userResourcePools.get(topologySubmitter))); } } return userMap; }
@Override public void schedule(Topologies topologies, Cluster cluster) { Map<String, User> userMap = getUsers(cluster); List<TopologyDetails> orderedTopologies = new ArrayList<>(schedulingPriorityStrategy.getOrderedTopologies(cluster, userMap)); if (LOG.isDebugEnabled()) { LOG.debug("Ordered list of topologies is: {}", orderedTopologies.stream().map((t) -> t.getId()).collect(Collectors.toList())); } for (TopologyDetails td : orderedTopologies) { if (!cluster.needsSchedulingRas(td)) { //cluster forgets about its previous status, so if it is scheduled just leave it. cluster.setStatusIfAbsent(td.getId(), "Fully Scheduled"); } else { User submitter = userMap.get(td.getTopologySubmitter()); scheduleTopology(td, cluster, submitter, orderedTopologies); } } }
TopologyDetails td = tds.get(id); if (td != null) { currentAssignment.set_owner(td.getTopologySubmitter()); state.setAssignment(id, currentAssignment, td.getConf());
String user = td.getTopologySubmitter(); LOG.debug("Found top {} run by user {}", td.getId(), user); NodePool pool = userPools.get(user);
newAssignment.set_owner(td.getTopologySubmitter()); newAssignments.put(topoId, newAssignment);
List<TopologyDetails> reversedList = ImmutableList.copyOf(orderedTopologies).reverse(); boolean evictedSomething = false; LOG.debug("attempting to make space for topo {} from user {}", td.getName(), td.getTopologySubmitter()); int tdIndex = reversedList.indexOf(td); double cpuNeeded = td.getTotalRequestedCpu(); topologyEvict.getTopologySubmitter()); cpuNeeded -= getCpuUsed(evictAssignemnt); memoryNeeded -= getMemoryUsed(evictAssignemnt);
private User cleanup(SchedulingState schedulingState, TopologyDetails td) { restoreCheckpointSchedulingState(schedulingState); //since state is restored need the update User topologySubmitter to the new User object in userMap return this.schedulingState.userMap.get(td.getTopologySubmitter()); }
private void evictTopology(TopologyDetails topologyEvict) { Collection<WorkerSlot> workersToEvict = this.cluster.getUsedSlotsByTopologyId(topologyEvict.getId()); User submitter = this.userMap.get(topologyEvict.getTopologySubmitter()); LOG.info("Evicting Topology {} with workers: {} from user {}", topologyEvict.getName(), workersToEvict, topologyEvict.getTopologySubmitter()); this.nodes.freeSlots(workersToEvict); submitter.moveTopoFromRunningToPending(topologyEvict, this.cluster); }
String topologySubmitter = td.getTopologySubmitter();
String user = td.getTopologySubmitter(); LOG.debug("Found top {} run by user {}",td.getId(), user); NodePool pool = userPools.get(user);
public void scheduleTopology(TopologyDetails td) { User topologySubmitter = this.schedulingState.userMap.get(td.getTopologySubmitter()); if (this.schedulingState.cluster.getUnassignedExecutors(td).size() > 0) { LOG.debug("/********Scheduling topology {} from User {}************/", td.getName(), topologySubmitter);
@Override public boolean makeSpaceForTopo(TopologyDetails td) { LOG.debug("attempting to make space for topo {} from user {}", td.getName(), td.getTopologySubmitter()); User submitter = this.userMap.get(td.getTopologySubmitter()); if (submitter.getCPUResourceGuaranteed() == null || submitter.getMemoryResourceGuaranteed() == null || submitter.getCPUResourceGuaranteed() == 0.0 || submitter.getMemoryResourceGuaranteed() == 0.0) {