public static void defaultSchedule(Topologies topologies, Cluster cluster) { for (TopologyDetails topology : cluster.needsSchedulingTopologies()) { List<WorkerSlot> availableSlots = cluster.getAvailableSlots(); Set<ExecutorDetails> allExecutors = topology.getExecutors(); Map<WorkerSlot, List<ExecutorDetails>> aliveAssigned = EvenScheduler.getAliveAssignedWorkerSlotExecutors(cluster, topology.getId()); Set<ExecutorDetails> aliveExecutors = new HashSet<ExecutorDetails>(); for (List<ExecutorDetails> list : aliveAssigned.values()) { aliveExecutors.addAll(list); } Set<WorkerSlot> canReassignSlots = slotsCanReassign(cluster, aliveAssigned.keySet()); int totalSlotsToUse = Math.min(topology.getNumWorkers(), canReassignSlots.size() + availableSlots.size()); Set<WorkerSlot> badSlots = null; if (totalSlotsToUse > aliveAssigned.size() || !allExecutors.equals(aliveExecutors)) { badSlots = badSlots(aliveAssigned, allExecutors.size(), totalSlotsToUse); } if (badSlots != null) { cluster.freeSlots(badSlots); } EvenScheduler.scheduleTopologiesEvenly(new Topologies(topology), cluster); } }
_topId = td.getId(); _cluster = cluster; Map<ExecutorDetails, String> execToComp = td.getExecutorToComponent(); SchedulerAssignment assignment = _cluster.getAssignmentById(_topId); _nodeToComps = new HashMap<>(); List<String> spreadComps = (List<String>) td.getConf().get(Config.TOPOLOGY_SPREAD_COMPONENTS); if (spreadComps != null) { for (String comp : spreadComps) {
@Override public String toString() { return "Name: " + getName() + " id: " + getId() + " Priority: " + getTopologyPriority() + " Uptime: " + getUpTime() + " CPU: " + getTotalRequestedCpu() + " Memory: " + (getTotalRequestedMemOffHeap() + getTotalRequestedMemOnHeap()); }
public double getRequestedNonSharedOffHeap() { double totalMemOffHeap = 0.0; for (ExecutorDetails exec : this.getExecutors()) { Double execMem = getOffHeapMemoryRequirement(exec); if (execMem != null) { totalMemOffHeap += execMem; } } return totalMemOffHeap; }
public double getRequestedNonSharedOnHeap() { double totalMemOnHeap = 0.0; for (ExecutorDetails exec : this.getExecutors()) { Double execMem = getOnHeapMemoryRequirement(exec); if (execMem != null) { totalMemOnHeap += execMem; } } return totalMemOnHeap; }
RAS_Nodes nodes = new RAS_Nodes(workingState); IStrategy rasStrategy = null; String strategyConf = (String) td.getConf().get(Config.TOPOLOGY_SCHEDULER_STRATEGY); try { String strategy = (String) td.getConf().get(Config.TOPOLOGY_SCHEDULER_STRATEGY); if (strategy.startsWith("backtype.storm")) { SingleTopologyCluster toSchedule = new SingleTopologyCluster(workingState, td.getId()); try { SchedulingResult result = rasStrategy.schedule(toSchedule, td); if (result.isSuccess()) { cluster.updateFrom(toSchedule); cluster.setStatus(td.getId(), "Running - " + result.getMessage()); LOG.debug("Not enough resources to schedule {}", td.getName()); List<TopologyDetails> reversedList = ImmutableList.copyOf(orderedTopologies).reverse(); boolean evictedSomething = false; LOG.debug("attempting to make space for topo {} from user {}", td.getName(), td.getTopologySubmitter()); int tdIndex = reversedList.indexOf(td); double cpuNeeded = td.getTotalRequestedCpu(); double memoryNeeded = td.getTotalRequestedMemOffHeap() + td.getTotalRequestedMemOnHeap(); SchedulerAssignment assignment = cluster.getAssignmentById(td.getId()); if (assignment != null) { cpuNeeded -= getCpuUsed(assignment); SchedulerAssignment evictAssignemnt = workingState.getAssignmentById(topologyEvict.getId()); if (evictAssignemnt != null && !evictAssignemnt.getSlots().isEmpty()) { Collection<WorkerSlot> workersToEvict = workingState.getUsedSlotsByTopologyId(topologyEvict.getId());
config1.putAll(defaultTopologyConf); Map<ExecutorDetails, String> executorMap1 = genExecsAndComps(stormTopology1); TopologyDetails topology1 = new TopologyDetails("topology1", config1, stormTopology1, 3, executorMap1, 0, "user"); TopologyDetails topology2 = new TopologyDetails("topology2", config2, stormTopology2, 2, executorMap2, 0, "user"); SchedulerAssignment assignment = cluster.getAssignmentById(topology2.getId()); SchedulerAssignment newAssignment = cluster.getAssignmentById(topology2.getId()); Map<ExecutorDetails, WorkerSlot> newExecutorToSlot = newAssignment.getExecutorToSlot(); assertEquals("Running - Fully Scheduled by DefaultResourceAwareStrategy", cluster.getStatusMap().get(topology2.getId())); executorToSlot.put(new ExecutorDetails(2, 2), new WorkerSlot("r000s001", 1)); Map<String, SchedulerAssignment> existingAssignments = new HashMap<>(); assignment = new SchedulerAssignmentImpl(topology1.getId(), executorToSlot, null, null); existingAssignments.put(topology1.getId(), assignment); copyOfOldMapping = new HashMap<>(executorToSlot); Set<ExecutorDetails> existingExecutors = copyOfOldMapping.keySet(); newAssignment = cluster1.getAssignmentById(topology1.getId()); newExecutorToSlot = newAssignment.getExecutorToSlot(); assertEquals("Fully Scheduled", cluster1.getStatusMap().get(topology1.getId())); executorToSlot.put(new ExecutorDetails(2, 2), new WorkerSlot("r000s001", 1)); // the healthy worker existingAssignments = new HashMap<>(); assignment = new SchedulerAssignmentImpl(topology1.getId(), executorToSlot, null, null); existingAssignments.put(topology1.getId(), assignment);
@Override public boolean equals(Object o) { if (!(o instanceof TopologyDetails)) { return false; } return (topologyId.equals(((TopologyDetails) o).getId())); } }
private static Map<ExecutorDetails, WorkerSlot> scheduleTopology(TopologyDetails topology, Cluster cluster) { List<WorkerSlot> availableSlots = cluster.getAvailableSlots(); Set<ExecutorDetails> allExecutors = topology.getExecutors(); Map<WorkerSlot, List<ExecutorDetails>> aliveAssigned = getAliveAssignedWorkerSlotExecutors(cluster, topology.getId()); int totalSlotsToUse = Math.min(topology.getNumWorkers(), availableSlots.size() + aliveAssigned.size()); LOG.error("No available slots for topology: {}", topology.getName()); return new HashMap<ExecutorDetails, WorkerSlot>();
assert (cluster.getAssignmentById(topo.getId()) != null); Map<ExecutorDetails, WorkerSlot> result = cluster.getAssignmentById(topo.getId()).getExecutorToSlot(); Map<RAS_Node, Collection<ExecutorDetails>> nodeToExecs = new HashMap<>(); Map<ExecutorDetails, WorkerSlot> mergedExecToWorker = new HashMap<>(); Map<String, RAS_Node> nodes = RAS_Nodes.getAllNodesFrom(cluster); if (cluster.getAssignmentById(topo.getId()) != null && cluster.getAssignmentById(topo.getId()).getExecutorToSlot() != null) { mergedExecToWorker.putAll(cluster.getAssignmentById(topo.getId()).getExecutorToSlot()); double memoryUsed = 0.0; for (ExecutorDetails exec : execs) { cpuUsed += topo.getTotalCpuReqTask(exec); memoryUsed += topo.getTotalMemReqTask(exec);
List<String> favoredNodesIds = makeHostToNodeIds((List<String>) td.getConf().get(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES)); List<String> unFavoredNodesIds = makeHostToNodeIds((List<String>) td.getConf().get(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES)); final Iterable<String> sortedNodes = sortAllNodes(td, null, favoredNodesIds, unFavoredNodesIds); "Attempting to schedule: {} of component {}[ REQ {} ]", exec, td.getExecutorToComponent().get(exec), td.getTaskResourceReqList(exec)); if (!scheduleExecutor(exec, td, scheduledTasks, sortedNodes)) { return mkNotEnoughResources(td); SchedulingResult.failure( SchedulingStatus.FAIL_NOT_ENOUGH_RESOURCES, (td.getExecutors().size() - unassignedExecutors.size()) + "/" + td.getExecutors().size() + " executors scheduled"); } else {
TopologyDetails td = _tds.get(topId); Set<Node> allNodes = _topologyIdToNodes.get(topId); Number nodesRequested = (Number) td.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES); Integer effectiveNodesRequested = null; if (nodesRequested != null) { effectiveNodesRequested = Math.min(td.getExecutors().size(), +nodesRequested.intValue()); Node n = findBestNode(allNodes); if (n == null) { LOG.error("No nodes to use to assign topology {}", td.getName()); break;
private void evictTopology(TopologyDetails topologyEvict) { Collection<WorkerSlot> workersToEvict = this.cluster.getUsedSlotsByTopologyId(topologyEvict.getId()); User submitter = this.userMap.get(topologyEvict.getTopologySubmitter()); LOG.info("Evicting Topology {} with workers: {} from user {}", topologyEvict.getName(), workersToEvict, topologyEvict.getTopologySubmitter()); this.nodes.freeSlots(workersToEvict); submitter.moveTopoFromRunningToPending(topologyEvict, this.cluster); }
config.putAll(defaultTopologyConf); Map<ExecutorDetails, String> executorMap1 = genExecsAndComps(stormTopology1); TopologyDetails topology1 = new TopologyDetails("topology1", config, stormTopology1, 2, executorMap1, 0, "user"); rs.schedule(topologies, cluster); SchedulerAssignment assignment1 = cluster.getAssignmentById(topology1.getId()); Set<WorkerSlot> assignedSlots1 = assignment1.getSlots(); Set<String> nodesIDs1 = new HashSet<>(); List<Double> assignedExecutorCpu = new ArrayList<>(); for (ExecutorDetails executor : executors1) { assignedExecutorMemory.add(topology1.getTotalMemReqTask(executor)); assignedExecutorCpu.add(topology1.getTotalCpuReqTask(executor)); Double supervisorUsedMemory = 0.0; for (ExecutorDetails executor : entry.getValue()) { supervisorUsedMemory += topology1.getTotalCpuReqTask(executor); supervisorTotalCpu += topology1.getTotalMemReqTask(executor); assertTrue(entry.getKey() - entry.getValue() >= 0); assertEquals("Running - Fully Scheduled by DefaultResourceAwareStrategy", cluster.getStatusMap().get(topology1.getId()));
@Override public Collection<ExecutorDetails> getUnassignedExecutors(TopologyDetails topology) { if (topology == null) { return new ArrayList<>(0); } Collection<ExecutorDetails> ret = new HashSet<>(topology.getExecutors()); SchedulerAssignment assignment = getAssignmentById(topology.getId()); if (assignment != null) { Set<ExecutorDetails> assignedExecutors = assignment.getExecutors(); ret.removeAll(assignedExecutors); } return ret; }
Set<String> missingAssignmentTopologies = new HashSet<>(); for (TopologyDetails topo : topologies.getTopologies()) { String id = topo.getId(); Set<List<Integer>> allExecs = topoToExec.get(id); Set<List<Integer>> aliveExecs = topoToAliveExecutors.get(id); int numDesiredWorkers = topo.getNumWorkers(); int numAssignedWorkers = numUsedWorkers(topoToSchedAssignment.get(id)); if (allExecs == null || allExecs.isEmpty() || !allExecs.equals(aliveExecs) || numDesiredWorkers > numAssignedWorkers) {
@Override public Map<ExecutorDetails, String> getNeedsSchedulingExecutorToComponents( TopologyDetails topology) { Collection<ExecutorDetails> allExecutors = new HashSet<>(topology.getExecutors()); SchedulerAssignment assignment = assignments.get(topology.getId()); if (assignment != null) { allExecutors.removeAll(assignment.getExecutors()); } return topology.selectExecutorToComponent(allExecutors); }
@Override public void addTopology(TopologyDetails td) { String topId = td.getId(); LOG.debug("Adding in Topology {}", topId); SchedulerAssignment assignment = _cluster.getAssignmentById(topId); Set<Node> assignedNodes = new HashSet<>(); if (assignment != null) { for (WorkerSlot ws : assignment.getSlots()) { Node n = _nodeIdToNode.get(ws.getNodeId()); assignedNodes.add(n); } } _usedNodes += assignedNodes.size(); _topologyIdToNodes.put(topId, assignedNodes); _tds.put(topId, td); if (td.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES) != null) { _isolated.add(topId); } }
for (ExecutorDetails exec : getExecutors()) { if (!resourceList.containsKey(exec)) { LOG.debug( "Scheduling {} {} with resource requirement as {}", getExecutorToComponent().get(exec), exec, topologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB), resourceList.get(exec) ); addDefaultResforExec(exec);