@Test(expected=IllegalArgumentException.class) public void testAllocationIncrementInvalidUnit() throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RESOURCE_TYPES + "." + ResourceInformation.MEMORY_MB.getName() + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "1 Xi"); new FairSchedulerConfiguration(conf).getIncrementAllocation(); }
@Before public void setUp() throws Exception { scheduler = new FairScheduler(); conf = new FairSchedulerConfiguration(); }
@Override protected Configuration createConfiguration() throws IOException { FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration();
@Override protected Configuration createConfiguration() throws IOException { FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration();
@Before public void setup() throws Exception { Configuration conf = new Configuration(); clock = new TestFairScheduler.MockClock(); scheduler = mock(FairScheduler.class); when(scheduler.getConf()).thenReturn( new FairSchedulerConfiguration(conf)); when(scheduler.getClock()).thenReturn(clock); AllocationConfiguration allocConf = new AllocationConfiguration( conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); queueManager = new QueueManager(scheduler); queueManager.initialize(conf); queueMaxApps = allocConf.queueMaxApps; userMaxApps = allocConf.userMaxApps; maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler); appNum = 0; rmContext = mock(RMContext.class); when(rmContext.getEpoch()).thenReturn(0L); }
@Before public void setUp() throws Exception { conf = new FairSchedulerConfiguration(); FairScheduler scheduler = mock(FairScheduler.class); AllocationConfiguration allocConf = new AllocationConfiguration(conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); SystemClock clock = new SystemClock(); when(scheduler.getClock()).thenReturn(clock); notEmptyQueues = new HashSet<FSQueue>(); queueManager = new QueueManager(scheduler) { @Override public boolean isEmpty(FSQueue queue) { return !notEmptyQueues.contains(queue); } }; FSQueueMetrics.forQueue("root", null, true, conf); queueManager.initialize(conf); }
@Before public void setUp() throws Exception { conf = new FairSchedulerConfiguration(); scheduler = mock(FairScheduler.class); AllocationConfiguration allocConf = new AllocationConfiguration(conf); // Set up some queues to test default child max resource inheritance allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.test"); allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.test.childA"); allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.test.childB"); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); when(scheduler.getResourceCalculator()).thenReturn( new DefaultResourceCalculator()); SystemClock clock = SystemClock.getInstance(); when(scheduler.getClock()).thenReturn(clock); notEmptyQueues = new HashSet<>(); queueManager = new QueueManager(scheduler) { @Override public boolean isEmpty(FSQueue queue) { return !notEmptyQueues.contains(queue); } }; FSQueueMetrics.forQueue("root", null, true, conf); queueManager.initialize(conf); }
@Before public void setUp() throws Exception { conf = new FairSchedulerConfiguration(); FairScheduler scheduler = mock(FairScheduler.class); AllocationConfiguration allocConf = new AllocationConfiguration(conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); when(scheduler.getResourceCalculator()).thenReturn( new DefaultResourceCalculator()); SystemClock clock = SystemClock.getInstance(); when(scheduler.getClock()).thenReturn(clock); notEmptyQueues = new HashSet<FSQueue>(); queueManager = new QueueManager(scheduler) { @Override public boolean isEmpty(FSQueue queue) { return !notEmptyQueues.contains(queue); } }; FSQueueMetrics.forQueue("root", null, true, conf); queueManager.initialize(conf); }
@Before public void setup() throws Exception { Configuration conf = new Configuration(); clock = new ControlledClock(); scheduler = mock(FairScheduler.class); when(scheduler.getConf()).thenReturn( new FairSchedulerConfiguration(conf)); when(scheduler.getClock()).thenReturn(clock); AllocationConfiguration allocConf = new AllocationConfiguration( conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getResourceCalculator()).thenReturn( new DefaultResourceCalculator()); queueManager = new QueueManager(scheduler); queueManager.initialize(conf); userMaxApps = allocConf.userMaxApps; maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler); appNum = 0; rmContext = mock(RMContext.class); when(rmContext.getEpoch()).thenReturn(0L); }
private static FairScheduler mockFairSchedulerWithoutApps(RMContext rmContext) throws IOException { FairScheduler fs = new FairScheduler() { @Override public FSAppAttempt getSchedulerApp(ApplicationAttemptId applicationAttemptId) { return null ; } @Override public FSAppAttempt getApplicationAttempt(ApplicationAttemptId applicationAttemptId) { return null; } }; FairSchedulerConfiguration conf = new FairSchedulerConfiguration(); fs.setRMContext(rmContext); fs.init(conf); return fs; }
private static FairScheduler mockFairSchedulerWithoutApps(RMContext rmContext) throws IOException { FairScheduler fs = new FairScheduler() { @Override public FSAppAttempt getSchedulerApp(ApplicationAttemptId applicationAttemptId) { return null ; } @Override public FSAppAttempt getApplicationAttempt(ApplicationAttemptId applicationAttemptId) { return null; } }; FairSchedulerConfiguration conf = new FairSchedulerConfiguration(); fs.setRMContext(rmContext); fs.init(conf); return fs; }
@Test public void testMemoryIncrementConfiguredViaMultipleProperties() { TestAppender testAppender = new TestAppender(); Log4JLogger logger = (Log4JLogger) FairSchedulerConfiguration.LOG; logger.getLogger().addAppender(testAppender); try { Configuration conf = new Configuration(); conf.set("yarn.scheduler.increment-allocation-mb", "7"); conf.set(YarnConfiguration.RESOURCE_TYPES + "." + ResourceInformation.MEMORY_MB.getName() + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "13"); FairSchedulerConfiguration fsc = new FairSchedulerConfiguration(conf); Resource increment = fsc.getIncrementAllocation(); Assert.assertEquals(13L, increment.getMemorySize()); assertTrue("Warning message is not logged when specifying memory " + "increment via multiple properties", testAppender.getLogEvents().stream().anyMatch( e -> e.getLevel() == Level.WARN && ("Configuration " + "yarn.resource-types.memory-mb.increment-allocation=13 is " + "overriding the yarn.scheduler.increment-allocation-mb=7 " + "property").equals(e.getMessage()))); } finally { logger.getLogger().removeAppender(testAppender); } }
@Test public void testCpuIncrementConfiguredViaMultipleProperties() { TestAppender testAppender = new TestAppender(); Log4JLogger logger = (Log4JLogger) FairSchedulerConfiguration.LOG; logger.getLogger().addAppender(testAppender); try { Configuration conf = new Configuration(); conf.set("yarn.scheduler.increment-allocation-vcores", "7"); conf.set(YarnConfiguration.RESOURCE_TYPES + "." + ResourceInformation.VCORES.getName() + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "13"); FairSchedulerConfiguration fsc = new FairSchedulerConfiguration(conf); Resource increment = fsc.getIncrementAllocation(); Assert.assertEquals(13, increment.getVirtualCores()); assertTrue("Warning message is not logged when specifying CPU vCores " + "increment via multiple properties", testAppender.getLogEvents().stream().anyMatch( e -> e.getLevel() == Level.WARN && ("Configuration " + "yarn.resource-types.vcores.increment-allocation=13 is " + "overriding the yarn.scheduler.increment-allocation-vcores=7 " + "property").equals(e.getMessage()))); } finally { logger.getLogger().removeAppender(testAppender); } } }
@Test public void testEmptyChildQueues() throws Exception { FairSchedulerConfiguration conf = new FairSchedulerConfiguration(); FairScheduler scheduler = mock(FairScheduler.class); AllocationConfiguration allocConf = new AllocationConfiguration(conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(1, 1)); when(scheduler.getResourceCalculator()).thenReturn( new DefaultResourceCalculator()); SystemClock clock = SystemClock.getInstance(); when(scheduler.getClock()).thenReturn(clock); QueueManager queueManager = new QueueManager(scheduler); queueManager.initialize(conf); FSQueue testQueue = queueManager.getLeafQueue("test", true); FairSchedulerQueueInfo queueInfo = new FairSchedulerQueueInfo(testQueue, scheduler); Collection<FairSchedulerQueueInfo> childQueues = queueInfo.getChildQueues(); Assert.assertNotNull(childQueues); Assert.assertEquals("Child QueueInfo was not empty", 0, childQueues.size()); } }
private static FairScheduler mockFairScheduler() throws IOException { FairScheduler fs = new FairScheduler(); FairSchedulerConfiguration conf = new FairSchedulerConfiguration(); fs.setRMContext(new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null)); fs.init(conf); return fs; }
private static FairScheduler mockFairScheduler() throws IOException { FairScheduler fs = new FairScheduler(); FairSchedulerConfiguration conf = new FairSchedulerConfiguration(); fs.setRMContext(new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null)); fs.init(conf); return fs; }
@Test public void testAllocationIncrementVCoreWithUnit() throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RESOURCE_TYPES + "." + ResourceInformation.VCORES.getName() + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "1k"); FairSchedulerConfiguration fsc = new FairSchedulerConfiguration(conf); Resource min = Resources.createResource(0L, 0); Resource max = Resources.createResource(Long.MAX_VALUE, Integer.MAX_VALUE); Resource increment = fsc.getIncrementAllocation(); DominantResourceCalculator resourceCalculator = new DominantResourceCalculator(); assertEquals(1000, resourceCalculator.normalize( Resources.createResource(0L, 999), min, max, increment) .getVirtualCores()); assertEquals(1000, resourceCalculator.normalize( Resources.createResource(0L, 1000), min, max, increment) .getVirtualCores()); assertEquals(2000, resourceCalculator.normalize( Resources.createResource(0L, 1001), min, max, increment) .getVirtualCores()); }
@Test public void testAllocationIncrementVCoreNoUnit() throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RESOURCE_TYPES + "." + ResourceInformation.VCORES.getName() + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "10"); FairSchedulerConfiguration fsc = new FairSchedulerConfiguration(conf); Resource min = Resources.createResource(0L, 0); Resource max = Resources.createResource(Long.MAX_VALUE, Integer.MAX_VALUE); Resource increment = fsc.getIncrementAllocation(); DominantResourceCalculator resourceCalculator = new DominantResourceCalculator(); assertEquals(10, resourceCalculator.normalize( Resources.createResource(0L, 9), min, max, increment) .getVirtualCores()); assertEquals(10, resourceCalculator.normalize( Resources.createResource(0L, 10), min, max, increment) .getVirtualCores()); assertEquals(20, resourceCalculator.normalize( Resources.createResource(0L, 11), min, max, increment) .getVirtualCores()); }
@Test public void testAllocationIncrementMemoryNonDefaultUnit() throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RESOURCE_TYPES + "." + ResourceInformation.MEMORY_MB.getName() + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "1 Gi"); FairSchedulerConfiguration fsc = new FairSchedulerConfiguration(conf); Resource minimum = Resources.createResource(0L, 0); Resource maximum = Resources.createResource(Long.MAX_VALUE, Integer.MAX_VALUE); Resource increment = fsc.getIncrementAllocation(); DominantResourceCalculator resourceCalculator = new DominantResourceCalculator(); assertEquals(1024L, resourceCalculator.normalize( Resources.createResource(1023L), minimum, maximum, increment) .getMemorySize()); assertEquals(1024L, resourceCalculator.normalize( Resources.createResource(1024L), minimum, maximum, increment) .getMemorySize()); assertEquals(2048L, resourceCalculator.normalize( Resources.createResource(1025L), minimum, maximum, increment) .getMemorySize()); }
@Test public void testAllocationIncrementMemoryDefaultUnit() throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RESOURCE_TYPES + "." + ResourceInformation.MEMORY_MB.getName() + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "256"); FairSchedulerConfiguration fsc = new FairSchedulerConfiguration(conf); Resource minimum = Resources.createResource(0L, 0); Resource maximum = Resources.createResource(Long.MAX_VALUE, Integer.MAX_VALUE); Resource increment = fsc.getIncrementAllocation(); DominantResourceCalculator resourceCalculator = new DominantResourceCalculator(); assertEquals(1024L, resourceCalculator.normalize( Resources.createResource(769L), minimum, maximum, increment) .getMemorySize()); assertEquals(1024L, resourceCalculator.normalize( Resources.createResource(1023L), minimum, maximum, increment) .getMemorySize()); assertEquals(1024L, resourceCalculator.normalize( Resources.createResource(1024L), minimum, maximum, increment) .getMemorySize()); assertEquals(1280L, resourceCalculator.normalize( Resources.createResource(1025L), minimum, maximum, increment) .getMemorySize()); }