/** * Copy Constructor. * @param other what to copy. */ public NormalizedResourceOffer(NormalizedResourceOffer other) { this.totalMemoryMb = other.totalMemoryMb; this.normalizedResources = new NormalizedResources(other.normalizedResources); }
/** * Add the resources in other to this. * * @param other the other Request to add to this. */ public void add(NormalizedResourceRequest other) { this.normalizedResources.add(other.normalizedResources); onHeap += other.onHeap; offHeap += other.offHeap; }
LOG.trace("Calculating min percentage used by. Used Mem: {} Total Mem: {}" + " Used Normalized Resources: {} Total Normalized Resources: {}", totalMemoryMb, usedMemoryMb, toNormalizedMap(), used.toNormalizedMap()); throwBecauseUsedIsNotSubsetOfTotal(used, totalMemoryMb, usedMemoryMb); double totalCpu = getTotalCpu(); if (used.getTotalCpu() > totalCpu) { throwBecauseUsedIsNotSubsetOfTotal(used, totalMemoryMb, usedMemoryMb); min = Math.min(min, used.getTotalCpu() / totalCpu); throwBecauseUsedIsNotSubsetOfTotal(used, totalMemoryMb, usedMemoryMb); throwBecauseUsedIsNotSubsetOfTotal(used, totalMemoryMb, usedMemoryMb);
/** * A simple sanity check to see if all of the resources in this would be large enough to hold the resources in other ignoring memory. It * does not check memory because with shared memory it is beyond the scope of this. * * @param other the resources that we want to check if they would fit in this. * @param thisTotalMemoryMb The total memory in MB of this * @param otherTotalMemoryMb The total memory in MB of other * @return true if it might fit, else false if it could not possibly fit. */ public boolean couldHoldIgnoringSharedMemory(NormalizedResources other, double thisTotalMemoryMb, double otherTotalMemoryMb) { if (this.cpu < other.getTotalCpu()) { return false; } int length = Math.max(this.otherResources.length, other.otherResources.length); for (int i = 0; i < length; i++) { if (getResourceAt(i) < other.getResourceAt(i)) { return false; } } return thisTotalMemoryMb >= otherTotalMemoryMb; }
double total = 0.0; if (usedMemoryMb > totalMemoryMb) { throwBecauseUsedIsNotSubsetOfTotal(used, totalMemoryMb, usedMemoryMb); skippedResourceTypes++; double totalCpu = getTotalCpu(); if (used.getTotalCpu() > getTotalCpu()) { throwBecauseUsedIsNotSubsetOfTotal(used, totalMemoryMb, usedMemoryMb); total += used.getTotalCpu() / getTotalCpu(); } else { skippedResourceTypes++; throwBecauseUsedIsNotSubsetOfTotal(used, totalMemoryMb, usedMemoryMb); throwBecauseUsedIsNotSubsetOfTotal(used, totalMemoryMb, usedMemoryMb);
@Test public void testCalculateMinUsageWithNoResourcesInTotal() { NormalizedResources resources = new NormalizedResources(normalize(Collections.emptyMap())); NormalizedResources usedResources = new NormalizedResources(normalize(Collections.emptyMap())); double min = resources.calculateMinPercentageUsedBy(usedResources, 0, 0); assertThat(min, is(100.0)); }
@Test public void testCalculateAvgWithUnusedResource() { Map<String, Double> allResourcesMap = new HashMap<>(); allResourcesMap.put(Constants.COMMON_CPU_RESOURCE_NAME, 2.0); allResourcesMap.put(gpuResourceName, 10.0); NormalizedResources resources = new NormalizedResources(normalize(allResourcesMap)); Map<String, Double> usedResourcesMap = new HashMap<>(); usedResourcesMap.put(Constants.COMMON_CPU_RESOURCE_NAME, 1.0); NormalizedResources usedResources = new NormalizedResources(normalize(usedResourcesMap)); double avg = resources.calculateAveragePercentageUsedBy(usedResources, 4, 1); //The resource that is not used should count as if it is being used 0% assertThat(avg, is((50.0 + 25.0)/3)); }
@Test public void testAddToExistingResource() { NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 1))); NormalizedResources addedResources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 1))); resources.add(addedResources); Map<String, Double> normalizedMap = resources.toNormalizedMap(); assertThat(normalizedMap.get(gpuResourceName), is(2.0)); }
@Test public void testAddCpu() { NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); NormalizedResources addedResources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); resources.add(addedResources); Map<String, Double> normalizedMap = resources.toNormalizedMap(); assertThat(normalizedMap.get(Constants.COMMON_CPU_RESOURCE_NAME), is(2.0)); assertThat(resources.getTotalCpu(), is(2.0)); }
@Test public void testRemoveFromCpu() { NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); NormalizedResources removedResources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); resources.remove(removedResources, new ResourceMetrics(new StormMetricsRegistry())); Map<String, Double> normalizedMap = resources.toNormalizedMap(); assertThat(normalizedMap.get(Constants.COMMON_CPU_RESOURCE_NAME), is(1.0)); assertThat(resources.getTotalCpu(), is(1.0)); }
@Test public void testCouldHoldWithTooLittleMemory() { NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 1))); NormalizedResources resourcesToCheck = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 1))); boolean couldHold = resources.couldHoldIgnoringSharedMemory(resourcesToCheck, 100, 200); assertThat(couldHold, is(false)); }
@Test public void testRemoveZeroesWhenResourcesBecomeNegative() { NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 1))); NormalizedResources removedResources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 2))); resources.remove(removedResources, new ResourceMetrics(new StormMetricsRegistry())); Map<String, Double> normalizedMap = resources.toNormalizedMap(); assertThat(normalizedMap.get(gpuResourceName), is(0.0)); }
@Test public void testRemoveZeroesWhenCpuBecomesNegative() { NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); NormalizedResources removedResources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); resources.remove(removedResources, new ResourceMetrics(new StormMetricsRegistry())); assertThat(resources.getTotalCpu(), is(0.0)); }
@Override public String toString() { return "Normalized resources: " + toNormalizedMap(); }
/** * Remove the resources of a worker from this. * * @param value the worker resources that should be removed from this. */ public boolean remove(WorkerResources value) { Map<String, Double> workerNormalizedResources = value.get_resources(); cpu -= workerNormalizedResources.getOrDefault(Constants.COMMON_CPU_RESOURCE_NAME, 0.0); return remove(RESOURCE_MAP_ARRAY_BRIDGE.translateToResourceArray(workerNormalizedResources)) || cpu < 0; }
public double getTotalCpu() { return this.normalizedResources.getTotalCpu(); }
/** * Check if resources might be able to fit. * @see NormalizedResources#couldHoldIgnoringSharedMemory(org.apache.storm.scheduler.resource.normalization.NormalizedResources, double, * double) */ public boolean couldHoldIgnoringSharedMemory(NormalizedResourcesWithMemory other) { return normalizedResources.couldHoldIgnoringSharedMemory( other.getNormalizedResources(), getTotalMemoryMb(), other.getTotalMemoryMb()); }
/** * Calculate the average percentage used. * @see NormalizedResources#calculateAveragePercentageUsedBy(org.apache.storm.scheduler.resource.normalization.NormalizedResources, * double, double) */ public double calculateAveragePercentageUsedBy(NormalizedResourceOffer used) { return normalizedResources.calculateAveragePercentageUsedBy( used.getNormalizedResources(), getTotalMemoryMb(), used.getTotalMemoryMb()); }
/** * Calculate the min percentage used of the resource. * @see NormalizedResources#calculateMinPercentageUsedBy(org.apache.storm.scheduler.resource.normalization.NormalizedResources, double, * double) */ public double calculateMinPercentageUsedBy(NormalizedResourceOffer used) { return normalizedResources.calculateMinPercentageUsedBy(used.getNormalizedResources(), getTotalMemoryMb(), used.getTotalMemoryMb()); }
@Override public boolean areAnyOverZero() { return onHeap > 0 || offHeap > 0 || normalizedResources.areAnyOverZero(); } }