public static Cluster nextCluster(Cluster cluster, Map<String, SupervisorDetails> supervisors, INimbus iNimbus, Map<String, Object> config, Topologies topologies) { Map<String, SchedulerAssignmentImpl> assignment; if (cluster == null) { assignment = new HashMap<>(); } else { assignment = TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()); } return new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supervisors, assignment, topologies, config); } }
/** * Remove the other resources from this. This is the same as subtracting the resources in other from this. * * @param other the resources we want removed. * @param resourceMetrics The resource related metrics * @return true if the resources would have gone negative, but were clamped to 0. */ public boolean remove(NormalizedResources other, ResourceMetrics resourceMetrics) { boolean ret = false; this.cpu -= other.cpu; if (cpu < 0.0) { ret = true; resourceMetrics.getNegativeResourceEventsMeter().mark(); cpu = 0.0; } int otherLength = other.otherResources.length; zeroPadOtherResourcesIfNecessary(otherLength); for (int i = 0; i < otherLength; i++) { otherResources[i] -= other.otherResources[i]; if (otherResources[i] < 0.0) { ret = true; resourceMetrics.getNegativeResourceEventsMeter().mark(); otherResources[i] = 0.0; } } return ret; }
public Cluster makeCluster(TopologyDetails topo) { Topologies topologies = new Topologies(topo); Map<String, SupervisorDetails> supMap = genSupervisors(4, 2, 120, 1200); return new Cluster(new INimbusTest(), new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, new Config()); }
/** * Remove the resources in other from this. * @param other the resources to be removed. * @param resourceMetrics The resource related metrics * @return true if one or more resources in other were larger than available resources in this, else false. */ public boolean remove(WorkerResources other, ResourceMetrics resourceMetrics) { boolean negativeResources = normalizedResources.remove(other); totalMemoryMb -= (other.get_mem_off_heap() + other.get_mem_on_heap()); if (totalMemoryMb < 0.0) { negativeResources = true; resourceMetrics.getNegativeResourceEventsMeter().mark(); totalMemoryMb = 0.0; } return negativeResources; }
@Test public void testRemoveFromCpu() { NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); NormalizedResources removedResources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); resources.remove(removedResources, new ResourceMetrics(new StormMetricsRegistry())); Map<String, Double> normalizedMap = resources.toNormalizedMap(); assertThat(normalizedMap.get(Constants.COMMON_CPU_RESOURCE_NAME), is(1.0)); assertThat(resources.getTotalCpu(), is(1.0)); }
/** * Remove the resources in other from this. * @param other the resources to be removed. * @param resourceMetrics The resource related metrics * @return true if one or more resources in other were larger than available resources in this, else false. */ public boolean remove(NormalizedResourcesWithMemory other, ResourceMetrics resourceMetrics) { boolean negativeResources = normalizedResources.remove(other.getNormalizedResources(), resourceMetrics); totalMemoryMb -= other.getTotalMemoryMb(); if (totalMemoryMb < 0.0) { negativeResources = true; resourceMetrics.getNegativeResourceEventsMeter().mark(); totalMemoryMb = 0.0; } return negativeResources; }
@Test public void testRemoveZeroesWhenResourcesBecomeNegative() { NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 1))); NormalizedResources removedResources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 2))); resources.remove(removedResources, new ResourceMetrics(new StormMetricsRegistry())); Map<String, Double> normalizedMap = resources.toNormalizedMap(); assertThat(normalizedMap.get(gpuResourceName), is(0.0)); }
@Test public void testRemoveFromExistingResources() { NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 15))); NormalizedResources removedResources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 1))); resources.remove(removedResources, new ResourceMetrics(new StormMetricsRegistry())); Map<String, Double> normalizedMap = resources.toNormalizedMap(); assertThat(normalizedMap.get(gpuResourceName), is(14.0)); }
@Test public void testRemoveZeroesWhenCpuBecomesNegative() { NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); NormalizedResources removedResources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); resources.remove(removedResources, new ResourceMetrics(new StormMetricsRegistry())); assertThat(resources.getTotalCpu(), is(0.0)); }
Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); rs.schedule(topologies, cluster);
@Test public void sanityTestOfScheduling() { INimbus iNimbus = new INimbusTest(); Map<String, SupervisorDetails> supMap = genSupervisors(1, 2, 400, 2000); Config config = new Config(); config.putAll(defaultTopologyConf); ResourceAwareScheduler rs = new ResourceAwareScheduler(); TopologyDetails topology1 = genTopology("topology1", config, 1, 1, 1, 1, 0, 0, "user"); Topologies topologies = new Topologies(topology1); Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); rs.prepare(config); rs.schedule(topologies, cluster); SchedulerAssignment assignment = cluster.getAssignmentById(topology1.getId()); Set<WorkerSlot> assignedSlots = assignment.getSlots(); Set<String> nodesIDs = new HashSet<>(); for (WorkerSlot slot : assignedSlots) { nodesIDs.add(slot.getNodeId()); } Collection<ExecutorDetails> executors = assignment.getExecutors(); assertEquals(1, assignedSlots.size()); assertEquals(1, nodesIDs.size()); assertEquals(2, executors.size()); assertEquals("Running - Fully Scheduled by DefaultResourceAwareStrategy", cluster.getStatusMap().get(topology1.getId())); }
Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
this.conf = conf; this.metricsRegistry = metricsRegistry; this.resourceMetrics = new ResourceMetrics(metricsRegistry); this.submitTopologyWithOptsCalls = metricsRegistry.registerMeter("nimbus:num-submitTopologyWithOpts-calls"); this.submitTopologyCalls = metricsRegistry.registerMeter("nimbus:num-submitTopology-calls");
Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
@Test public void testNodeOverExtendedMemory() { NormalizedResourceOffer availableResources = createOffer(0.0, 5.0); NormalizedResourceOffer scheduledResources = createOffer(0.0, 10.0); availableResources.remove(scheduledResources, new ResourceMetrics(new StormMetricsRegistry())); Assert.assertEquals(0.0, availableResources.getTotalMemoryMb(), 0.001); }
@Test public void testNodeOverExtendedCpu() { NormalizedResourceOffer availableResources = createOffer(100.0, 0.0); NormalizedResourceOffer scheduledResources = createOffer(110.0, 0.0); availableResources.remove(scheduledResources, new ResourceMetrics(new StormMetricsRegistry())); Assert.assertEquals(0.0, availableResources.getTotalCpu(), 0.001); }
genTopology("topo-3-bobby", config, 1, 0, 1, 0,Time.currentTimeSecs() - 300,20, "bobby"), genTopology("topo-4-derek", config, 1, 0, 1, 0,Time.currentTimeSecs() - 201,29, "derek")); Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); genTopology("topo-5-derek", config, 1, 0, 1, 0,Time.currentTimeSecs() - 15,29, "derek")); cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); rs.schedule(topologies, cluster); genTopology("topo-6-bobby", config, 1, 0, 1, 0,Time.currentTimeSecs() - 10,29, "bobby")); cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); rs.schedule(topologies, cluster);
genTopology("topo-1", config, 1, 0, 2, 0, currentTime - 2, 29, "user"), genTopology("topo-2", config, 1, 0, 2, 0, currentTime - 2, 10, "user")); Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); ResourceAwareScheduler rs = new ResourceAwareScheduler();