/** * Add default resource requirements for a executor. */ private void addDefaultResforExec(ExecutorDetails exec) { String componentId = getExecutorToComponent().get(exec); addResourcesForExec(exec, new NormalizedResourceRequest(topologyConf, componentId)); }
private Set<Set<ExecutorDetails>> computeWorkerSpecs(TopologyDetails topology) { Map<String, List<ExecutorDetails>> compExecutors = Utils.reverseMap(topology.getExecutorToComponent()); List<ExecutorDetails> allExecutors = new ArrayList<ExecutorDetails>(); Collection<List<ExecutorDetails>> values = compExecutors.values(); for (List<ExecutorDetails> eList : values) { allExecutors.addAll(eList); } int numWorkers = topology.getNumWorkers(); int bucketIndex = 0; Map<Integer, Set<ExecutorDetails>> bucketExecutors = new HashMap<Integer, Set<ExecutorDetails>>(numWorkers); for (ExecutorDetails executor : allExecutors) { Set<ExecutorDetails> executors = bucketExecutors.get(bucketIndex); if (executors == null) { executors = new HashSet<ExecutorDetails>(); bucketExecutors.put(bucketIndex, executors); } executors.add(executor); bucketIndex = (bucketIndex + 1) % numWorkers; } return new HashSet<Set<ExecutorDetails>>(bucketExecutors.values()); }
/** * Check if constraints are satisfied. */ private static boolean checkConstraintsSatisfied(Cluster cluster, TopologyDetails topo) { LOG.info("Checking constraints..."); assert (cluster.getAssignmentById(topo.getId()) != null); Map<ExecutorDetails, WorkerSlot> result = cluster.getAssignmentById(topo.getId()).getExecutorToSlot(); Map<ExecutorDetails, String> execToComp = topo.getExecutorToComponent(); //get topology constraints Map<String, Map<String, Integer>> constraintMatrix = getConstraintMap(topo, new HashSet<>(topo.getExecutorToComponent().values())); Map<WorkerSlot, Set<String>> workerCompMap = new HashMap<>(); result.forEach((exec, worker) -> { String comp = execToComp.get(exec); workerCompMap.computeIfAbsent(worker, (k) -> new HashSet<>()).add(comp); }); for (Map.Entry<WorkerSlot, Set<String>> entry : workerCompMap.entrySet()) { Set<String> comps = entry.getValue(); for (String comp1 : comps) { for (String comp2 : comps) { if (!comp1.equals(comp2) && constraintMatrix.get(comp1).get(comp2) != 0) { LOG.error("Incorrect Scheduling: worker exclusion for Component {} and {} not satisfied on WorkerSlot: {}", comp1, comp2, entry.getKey()); return false; } } } } return true; }
assert (cluster.getAssignmentById(topo.getId()) != null); Map<ExecutorDetails, WorkerSlot> result = cluster.getAssignmentById(topo.getId()).getExecutorToSlot(); Map<ExecutorDetails, String> execToComp = topo.getExecutorToComponent(); Map<WorkerSlot, HashSet<ExecutorDetails>> workerExecMap = new HashMap<>(); Map<WorkerSlot, HashSet<String>> workerCompMap = new HashMap<>();
_cluster = cluster; Map<ExecutorDetails, String> execToComp = td.getExecutorToComponent(); SchedulerAssignment assignment = _cluster.getAssignmentById(_topId); _nodeToComps = new HashMap<>();
LOG.debug( "Scheduling {} {} with resource requirement as {}", getExecutorToComponent().get(exec), exec, topologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB),
"Attempting to schedule: {} of component {}[ REQ {} ]", exec, td.getExecutorToComponent().get(exec), td.getTaskResourceReqList(exec)); final Iterable<String> sortedNodes = sortAllNodes(td, exec, favoredNodeIds, unFavoredNodeIds);
"Attempting to schedule: {} of component {}[ REQ {} ]", exec, td.getExecutorToComponent().get(exec), td.getTaskResourceReqList(exec)); if (!scheduleExecutor(exec, td, scheduledTasks, sortedNodes)) {
execToComp = td.getExecutorToComponent();
return true; } else { String comp = td.getExecutorToComponent().get(exec); NormalizedResourceRequest requestedResources = td.getTotalResources(exec); LOG.error("Not Enough Resources to schedule Task {} - {} {}", exec, comp, requestedResources);
_cluster = cluster; Map<ExecutorDetails, String> execToComp = td.getExecutorToComponent(); SchedulerAssignment assignment = _cluster.getAssignmentById(_topId); _nodeToComps = new HashMap<>();
LOG.debug( "Scheduling {} {} with memory requirement as 'on heap' - {} and 'off heap' - {} and CPU requirement as {}", this.getExecutorToComponent().get(exec), exec, this.topologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB),
exec, td.getExecutorToComponent().get(exec), td.getTaskResourceReqList(exec)); scheduleExecutor(exec, td, schedulerAssignmentMap, scheduledTasks);