@Test(expected=IllegalArgumentException.class) public void testAllocationIncrementInvalidUnit() throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RESOURCE_TYPES + "." + ResourceInformation.MEMORY_MB.getName() + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "1 Xi"); new FairSchedulerConfiguration(conf).getIncrementAllocation(); }
long incrementMem = config.getIncrementAllocation().getMemorySize(); if (incrementMem <= 0) { throw new YarnRuntimeException("Invalid resource scheduler memory" int incrementVcore = config.getIncrementAllocation().getVirtualCores(); if (incrementVcore <= 0) { throw new YarnRuntimeException("Invalid resource scheduler vcores"
@Test public void testMemoryIncrementConfiguredViaMultipleProperties() { TestAppender testAppender = new TestAppender(); Log4JLogger logger = (Log4JLogger) FairSchedulerConfiguration.LOG; logger.getLogger().addAppender(testAppender); try { Configuration conf = new Configuration(); conf.set("yarn.scheduler.increment-allocation-mb", "7"); conf.set(YarnConfiguration.RESOURCE_TYPES + "." + ResourceInformation.MEMORY_MB.getName() + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "13"); FairSchedulerConfiguration fsc = new FairSchedulerConfiguration(conf); Resource increment = fsc.getIncrementAllocation(); Assert.assertEquals(13L, increment.getMemorySize()); assertTrue("Warning message is not logged when specifying memory " + "increment via multiple properties", testAppender.getLogEvents().stream().anyMatch( e -> e.getLevel() == Level.WARN && ("Configuration " + "yarn.resource-types.memory-mb.increment-allocation=13 is " + "overriding the yarn.scheduler.increment-allocation-mb=7 " + "property").equals(e.getMessage()))); } finally { logger.getLogger().removeAppender(testAppender); } }
@Test public void testCpuIncrementConfiguredViaMultipleProperties() { TestAppender testAppender = new TestAppender(); Log4JLogger logger = (Log4JLogger) FairSchedulerConfiguration.LOG; logger.getLogger().addAppender(testAppender); try { Configuration conf = new Configuration(); conf.set("yarn.scheduler.increment-allocation-vcores", "7"); conf.set(YarnConfiguration.RESOURCE_TYPES + "." + ResourceInformation.VCORES.getName() + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "13"); FairSchedulerConfiguration fsc = new FairSchedulerConfiguration(conf); Resource increment = fsc.getIncrementAllocation(); Assert.assertEquals(13, increment.getVirtualCores()); assertTrue("Warning message is not logged when specifying CPU vCores " + "increment via multiple properties", testAppender.getLogEvents().stream().anyMatch( e -> e.getLevel() == Level.WARN && ("Configuration " + "yarn.resource-types.vcores.increment-allocation=13 is " + "overriding the yarn.scheduler.increment-allocation-vcores=7 " + "property").equals(e.getMessage()))); } finally { logger.getLogger().removeAppender(testAppender); } } }
minimumAllocation = super.getMinimumAllocation(); initMaximumResourceCapability(super.getMaximumAllocation()); incrAllocation = this.conf.getIncrementAllocation(); updateReservationThreshold(); continuousSchedulingEnabled = this.conf.isContinuousSchedulingEnabled();
minimumAllocation = this.conf.getMinimumAllocation(); initMaximumResourceCapability(this.conf.getMaximumAllocation()); incrAllocation = this.conf.getIncrementAllocation(); continuousSchedulingEnabled = this.conf.isContinuousSchedulingEnabled(); continuousSchedulingSleepMs =
minimumAllocation = this.conf.getMinimumAllocation(); initMaximumResourceCapability(this.conf.getMaximumAllocation()); incrAllocation = this.conf.getIncrementAllocation(); continuousSchedulingEnabled = this.conf.isContinuousSchedulingEnabled(); continuousSchedulingSleepMs =
@Test public void testAllocationIncrementVCoreWithUnit() throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RESOURCE_TYPES + "." + ResourceInformation.VCORES.getName() + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "1k"); FairSchedulerConfiguration fsc = new FairSchedulerConfiguration(conf); Resource min = Resources.createResource(0L, 0); Resource max = Resources.createResource(Long.MAX_VALUE, Integer.MAX_VALUE); Resource increment = fsc.getIncrementAllocation(); DominantResourceCalculator resourceCalculator = new DominantResourceCalculator(); assertEquals(1000, resourceCalculator.normalize( Resources.createResource(0L, 999), min, max, increment) .getVirtualCores()); assertEquals(1000, resourceCalculator.normalize( Resources.createResource(0L, 1000), min, max, increment) .getVirtualCores()); assertEquals(2000, resourceCalculator.normalize( Resources.createResource(0L, 1001), min, max, increment) .getVirtualCores()); }
@Test public void testAllocationIncrementVCoreNoUnit() throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RESOURCE_TYPES + "." + ResourceInformation.VCORES.getName() + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "10"); FairSchedulerConfiguration fsc = new FairSchedulerConfiguration(conf); Resource min = Resources.createResource(0L, 0); Resource max = Resources.createResource(Long.MAX_VALUE, Integer.MAX_VALUE); Resource increment = fsc.getIncrementAllocation(); DominantResourceCalculator resourceCalculator = new DominantResourceCalculator(); assertEquals(10, resourceCalculator.normalize( Resources.createResource(0L, 9), min, max, increment) .getVirtualCores()); assertEquals(10, resourceCalculator.normalize( Resources.createResource(0L, 10), min, max, increment) .getVirtualCores()); assertEquals(20, resourceCalculator.normalize( Resources.createResource(0L, 11), min, max, increment) .getVirtualCores()); }
@Test public void testAllocationIncrementMemoryNonDefaultUnit() throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RESOURCE_TYPES + "." + ResourceInformation.MEMORY_MB.getName() + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "1 Gi"); FairSchedulerConfiguration fsc = new FairSchedulerConfiguration(conf); Resource minimum = Resources.createResource(0L, 0); Resource maximum = Resources.createResource(Long.MAX_VALUE, Integer.MAX_VALUE); Resource increment = fsc.getIncrementAllocation(); DominantResourceCalculator resourceCalculator = new DominantResourceCalculator(); assertEquals(1024L, resourceCalculator.normalize( Resources.createResource(1023L), minimum, maximum, increment) .getMemorySize()); assertEquals(1024L, resourceCalculator.normalize( Resources.createResource(1024L), minimum, maximum, increment) .getMemorySize()); assertEquals(2048L, resourceCalculator.normalize( Resources.createResource(1025L), minimum, maximum, increment) .getMemorySize()); }
@Test public void testAllocationIncrementMemoryDefaultUnit() throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RESOURCE_TYPES + "." + ResourceInformation.MEMORY_MB.getName() + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "256"); FairSchedulerConfiguration fsc = new FairSchedulerConfiguration(conf); Resource minimum = Resources.createResource(0L, 0); Resource maximum = Resources.createResource(Long.MAX_VALUE, Integer.MAX_VALUE); Resource increment = fsc.getIncrementAllocation(); DominantResourceCalculator resourceCalculator = new DominantResourceCalculator(); assertEquals(1024L, resourceCalculator.normalize( Resources.createResource(769L), minimum, maximum, increment) .getMemorySize()); assertEquals(1024L, resourceCalculator.normalize( Resources.createResource(1023L), minimum, maximum, increment) .getMemorySize()); assertEquals(1024L, resourceCalculator.normalize( Resources.createResource(1024L), minimum, maximum, increment) .getMemorySize()); assertEquals(1280L, resourceCalculator.normalize( Resources.createResource(1025L), minimum, maximum, increment) .getMemorySize()); }
FairSchedulerConfiguration.INCREMENT_ALLOCATION, "10"); FairSchedulerConfiguration fsc = new FairSchedulerConfiguration(conf); Resource increment = fsc.getIncrementAllocation(); DominantResourceCalculator calculator = new DominantResourceCalculator();