private static HashSet<String> getSpreadComps(TopologyDetails topo) { HashSet<String> retSet = new HashSet<>(); List<String> spread = (List<String>) topo.getConf().get(Config.TOPOLOGY_SPREAD_COMPONENTS); if (spread != null) { Set<String> comps = topo.getComponents().keySet(); for (String comp : spread) { if (comps.contains(comp)) { retSet.add(comp); } else { LOG.warn("Comp {} declared for spread not valid", comp); } } } return retSet; }
static Map<String, Map<String, Integer>> getConstraintMap(TopologyDetails topo, Set<String> comps) { Map<String, Map<String, Integer>> matrix = new HashMap<>(); for (String comp : comps) { matrix.put(comp, new HashMap<>()); for (String comp2 : comps) { matrix.get(comp).put(comp2, 0); } } List<List<String>> constraints = (List<List<String>>) topo.getConf().get(Config.TOPOLOGY_RAS_CONSTRAINTS); if (constraints != null) { for (List<String> constraintPair : constraints) { String comp1 = constraintPair.get(0); String comp2 = constraintPair.get(1); if (!matrix.containsKey(comp1)) { LOG.warn("Comp: {} declared in constraints is not valid!", comp1); continue; } if (!matrix.containsKey(comp2)) { LOG.warn("Comp: {} declared in constraints is not valid!", comp2); continue; } matrix.get(comp1).put(comp2, 1); matrix.get(comp2).put(comp1, 1); } } return matrix; }
@Override public void addTopology(TopologyDetails td) { String topId = td.getId(); LOG.debug("Adding in Topology {}", topId); SchedulerAssignment assignment = _cluster.getAssignmentById(topId); Set<Node> assignedNodes = new HashSet<>(); if (assignment != null) { for (WorkerSlot ws : assignment.getSlots()) { Node n = _nodeIdToNode.get(ws.getNodeId()); assignedNodes.add(n); } } _usedNodes += assignedNodes.size(); _topologyIdToNodes.put(topId, assignedNodes); _tds.put(topId, td); if (td.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES) != null) { _isolated.add(topId); } }
List<String> spreadComps = (List<String>) td.getConf().get(Config.TOPOLOGY_SPREAD_COMPONENTS); if (spreadComps != null) { for (String comp : spreadComps) {
TopologyDetails td = _tds.get(topId); Set<Node> allNodes = _topologyIdToNodes.get(topId); Number nodesRequested = (Number) td.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES); Integer effectiveNodesRequested = null; if (nodesRequested != null) {
ObjectReader.getInt(td.getConf().get(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_STATE_SEARCH), DEFAULT_STATE_SEARCH)); ObjectReader.getInt(td.getConf().get(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_TIME_SECS), -1).intValue() * 1000L; favoredNodeIds = makeHostToNodeIds((List<String>) td.getConf().get(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES)); unFavoredNodeIds = makeHostToNodeIds((List<String>) td.getConf().get(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES));
if (td != null) { currentAssignment.set_owner(td.getTopologySubmitter()); state.setAssignment(id, currentAssignment, td.getConf());
List<String> favoredNodesIds = makeHostToNodeIds((List<String>) td.getConf().get(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES)); List<String> unFavoredNodesIds = makeHostToNodeIds((List<String>) td.getConf().get(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES)); final Iterable<String> sortedNodes = sortAllNodes(td, null, favoredNodesIds, unFavoredNodesIds);
} else { LOG.info("Setting new assignment for topology id {}: {}", topoId, assignment); state.setAssignment(topoId, assignment, td.getConf());
int slots = 0; try { slots = ServerUtils.getEstimatedWorkerCountForRASTopo(td.getConf(), td.getTopology()); } catch (InvalidTopologyException e) { LOG.warn("Could not guess the number of slots needed for {}", td.getName(), e);
public static Map getFullTopologyConfig(Map conf, TopologyDetails info) { Map ret = new HashMap(conf); ret.putAll(info.getConf()); return ret; }
@Override public void addTopology(TopologyDetails td) { String topId = td.getId(); LOG.debug("Adding in Topology {}", topId); SchedulerAssignment assignment = _cluster.getAssignmentById(topId); Set<Node> assignedNodes = new HashSet<>(); if (assignment != null) { for (WorkerSlot ws: assignment.getSlots()) { Node n = _nodeIdToNode.get(ws.getNodeId()); assignedNodes.add(n); } } _usedNodes += assignedNodes.size(); _topologyIdToNodes.put(topId, assignedNodes); _tds.put(topId, td); if (td.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES) != null) { _isolated.add(topId); } }
List<String> spreadComps = (List<String>)td.getConf().get(Config.TOPOLOGY_SPREAD_COMPONENTS); if (spreadComps != null) { for (String comp: spreadComps) {
continue; Map topConf = topologies.getById(topId).getConf(); Double assignedMemForTopology = 0.0; Double assignedMemPerSlot = getAssignedMemoryForSlot(topConf);
continue; Map topConf = topologies.getById(topId).getConf(); Double assignedCpuForTopology = 0.0;
TopologyDetails td = _tds.get(topId); Set<Node> allNodes = _topologyIdToNodes.get(topId); Number nodesRequested = (Number) td.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES); Integer effectiveNodesRequested = null; if (nodesRequested != null) {
IStrategy rasStrategy = null; try { rasStrategy = (IStrategy) Utils.newInstance((String) td.getConf().get(Config.TOPOLOGY_SCHEDULER_STRATEGY)); } catch (RuntimeException e) { LOG.error("failed to create instance of IStrategy: {} with error: {}! Topology {} will not be scheduled.", td.getName(), td.getConf().get(Config.TOPOLOGY_SCHEDULER_STRATEGY), e.getMessage()); topologySubmitter = cleanup(schedulingState, td); topologySubmitter.moveTopoFromPendingToInvalid(td); this.schedulingState.cluster.setStatus(td.getId(), "Unsuccessful in scheduling - failed to create instance of topology strategy " + td.getConf().get(Config.TOPOLOGY_SCHEDULER_STRATEGY) + ". Please check logs for details"); return;