private static void setResourcesDefaultIfNotSet(Map<String, NormalizedResourceRequest> compResourcesMap, String compId, Map<String, Object> topoConf) { NormalizedResourceRequest resources = compResourcesMap.get(compId); if (resources == null) { compResourcesMap.put(compId, new NormalizedResourceRequest(topoConf, compId)); } }
@Override public String toString() { return "Normalized resources: " + toNormalizedMap(); }
/** * This medhod updates ScheduledResources and UsedSlots cache for given workerSlot. */ private void updateCachesForWorkerSlot(WorkerSlot workerSlot, WorkerResources workerResources, Double sharedoffHeapMemory) { String nodeId = workerSlot.getNodeId(); NormalizedResourceRequest normalizedResourceRequest = new NormalizedResourceRequest(); normalizedResourceRequest.add(workerResources); normalizedResourceRequest.addOffHeap(sharedoffHeapMemory); nodeToScheduledResourcesCache.computeIfAbsent(nodeId, MAKE_MAP).put(workerSlot, normalizedResourceRequest); nodeToUsedSlotsCache.computeIfAbsent(nodeId, MAKE_SET).add(workerSlot); }
/** * Get an approximate total resources needed for this topology. * @return the approximate total resources needed for this topology. */ public NormalizedResourceRequest getApproximateTotalResources() { NormalizedResourceRequest ret = new NormalizedResourceRequest(); for (NormalizedResourceRequest resources : resourceList.values()) { ret.add(resources); } return ret; }
public NormalizedResourceRequest(ComponentCommon component, Map<String, Object> topoConf, String componentId) { this(parseResources(component.get_json_conf()), getDefaultResources(topoConf, componentId)); }
private WorkerResources calculateWorkerResources( TopologyDetails td, Collection<ExecutorDetails> executors) { NormalizedResourceRequest totalResources = new NormalizedResourceRequest(); Map<String, Double> sharedTotalResources = new HashMap<>(); for (ExecutorDetails exec : executors) { continue; totalResources.add(allResources); totalResources.addOffHeap(shared.get_off_heap_worker()); totalResources.addOnHeap(shared.get_off_heap_worker()); ret.set_resources(totalResources.toNormalizedMap()); ret.set_shared_resources(sharedTotalResources); ret.set_cpu(totalResources.getTotalCpu()); ret.set_mem_off_heap(totalResources.getOffHeapMemoryMb()); ret.set_mem_on_heap(totalResources.getOnHeapMemoryMb()); ret.set_shared_mem_off_heap( sharedTotalResources.getOrDefault(
@Test public void testNonAckerCPUSetting() { Map<String, Object> topoConf = new HashMap<>(); topoConf.put(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT, 40); topoConf.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 50); NormalizedResourceRequest request = new NormalizedResourceRequest(topoConf, "notAnAckerComponent"); Map<String, Double> normalizedMap = request.toNormalizedMap(); Double cpu = normalizedMap.get(Constants.COMMON_CPU_RESOURCE_NAME); Assert.assertNotNull(cpu); Assert.assertEquals(50, cpu, 0.001); } }
/** * Get the total CPU requirement for executor. * * @param exec * @return Map<String , Double> generic resource mapping requirement for the executor */ public Double getTotalCpuReqTask(ExecutorDetails exec) { if (hasExecInTopo(exec)) { return resourceList .get(exec).getTotalCpu(); } return null; }
/** * Gets the on heap memory requirement for a certain task within a topology. * * @param exec the executor the inquiry is concerning. * @return Double the amount of on heap memory requirement for this exec in topology topoId. */ public Double getOnHeapMemoryRequirement(ExecutorDetails exec) { Double ret = null; if (hasExecInTopo(exec)) { ret = resourceList .get(exec).getOnHeapMemoryMb(); ; } return ret; }
public NormalizedResourceRequest(Map<String, Object> topoConf, String componentId) { this((Map<String, ? extends Number>) null, getDefaultResources(topoConf, componentId)); }
/** * If a node or rack has a kind of resource not in a request, make that resource negative so when sorting that node or rack will * be less likely to be selected. * @param requestedResources the requested resources. */ public void updateForRareResourceAffinity(NormalizedResourceRequest requestedResources) { normalizedResources.updateForRareResourceAffinity(requestedResources.getNormalizedResources()); }
/** * Gets the off heap memory requirement for a certain task within a topology. * * @param exec the executor the inquiry is concerning. * @return Double the amount of off heap memory requirement for this exec in topology topoId. */ public Double getOffHeapMemoryRequirement(ExecutorDetails exec) { Double ret = null; if (hasExecInTopo(exec)) { ret = resourceList .get(exec).getOffHeapMemoryMb(); } return ret; }
@Test public void testAckerCPUSetting() { Map<String, Object> topoConf = new HashMap<>(); topoConf.put(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT, 40); topoConf.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 50); NormalizedResourceRequest request = new NormalizedResourceRequest(topoConf, Acker.ACKER_COMPONENT_ID); Map<String, Double> normalizedMap = request.toNormalizedMap(); Double cpu = normalizedMap.get(Constants.COMMON_CPU_RESOURCE_NAME); Assert.assertNotNull(cpu); Assert.assertEquals(40, cpu, 0.001); }
@Override public NormalizedResourceRequest getAllScheduledResourcesForNode(String nodeId) { return totalResourcesPerNodeCache.computeIfAbsent(nodeId, (nid) -> { NormalizedResourceRequest totalScheduledResources = new NormalizedResourceRequest(); for (NormalizedResourceRequest req : nodeToScheduledResourcesCache.computeIfAbsent(nodeId, MAKE_MAP).values()) { totalScheduledResources.add(req); } return totalScheduledResources; }); }
@Override public double getScheduledCpuForNode(String nodeId) { return getAllScheduledResourcesForNode(nodeId).getTotalCpu(); }
public static double getEstimatedTotalHeapMemoryRequiredByTopo(Map<String, Object> topoConf, StormTopology topology) throws InvalidTopologyException { Map<String, Integer> componentParallelism = getComponentParallelism(topoConf, topology); double totalMemoryRequired = 0.0; for (Map.Entry<String, NormalizedResourceRequest> entry : ResourceUtils.getBoltsResources(topology, topoConf).entrySet()) { int parallelism = componentParallelism.getOrDefault(entry.getKey(), 1); double memoryRequirement = entry.getValue().getOnHeapMemoryMb(); totalMemoryRequired += memoryRequirement * parallelism; } for (Map.Entry<String, NormalizedResourceRequest> entry : ResourceUtils.getSpoutsResources(topology, topoConf).entrySet()) { int parallelism = componentParallelism.getOrDefault(entry.getKey(), 1); double memoryRequirement = entry.getValue().getOnHeapMemoryMb(); totalMemoryRequired += memoryRequirement * parallelism; } return totalMemoryRequired; }
NormalizedResourceRequest spoutResources = ResourceUtils.getSpoutResources(topology, topoConf, componentId); if (spoutResources == null) { spoutResources = new NormalizedResourceRequest(topoConf, componentId); compPageInfo.set_resources_map(spoutResources.toNormalizedMap()); } else { //bolt NormalizedResourceRequest boltResources = ResourceUtils.getBoltResources(topology, topoConf, componentId); if (boltResources == null) { boltResources = new NormalizedResourceRequest(topoConf, componentId); compPageInfo.set_resources_map(boltResources.toNormalizedMap());
/** * Add default resource requirements for a executor. */ private void addDefaultResforExec(ExecutorDetails exec) { String componentId = getExecutorToComponent().get(exec); addResourcesForExec(exec, new NormalizedResourceRequest(topologyConf, componentId)); }
CommonAggregateStats commonStats = entry.getValue().get_common_stats(); setResourcesDefaultIfNotSet(spoutResources, entry.getKey(), topoConf); commonStats.set_resources_map(spoutResources.get(entry.getKey()).toNormalizedMap()); CommonAggregateStats commonStats = entry.getValue().get_common_stats(); setResourcesDefaultIfNotSet(boltResources, entry.getKey(), topoConf); commonStats.set_resources_map(boltResources.get(entry.getKey()).toNormalizedMap());
public static Map<String, NormalizedResourceRequest> getBoltsResources(StormTopology topology, Map<String, Object> topologyConf) { Map<String, NormalizedResourceRequest> boltResources = new HashMap<>(); if (topology.get_bolts() != null) { for (Map.Entry<String, Bolt> bolt : topology.get_bolts().entrySet()) { NormalizedResourceRequest topologyResources = new NormalizedResourceRequest(bolt.getValue().get_common(), topologyConf, bolt.getKey()); if (LOG.isTraceEnabled()) { LOG.trace("Turned {} into {}", bolt.getValue().get_common().get_json_conf(), topologyResources); } boltResources.put(bolt.getKey(), topologyResources); } } return boltResources; }