private FSLeafQueue getLeafQueue(String name, boolean create, ApplicationId applicationId, boolean recomputeSteadyShares) { FSQueue queue = getQueue(name, create, FSQueueType.LEAF, recomputeSteadyShares, applicationId); if (queue instanceof FSParentQueue) { return null; } return (FSLeafQueue) queue; }
public FairScheduler() { super(FairScheduler.class.getName()); context = new FSContext(this); allocsLoader = new AllocationFileLoaderService(); queueMgr = new QueueManager(this); maxRunningEnforcer = new MaxRunningAppsEnforcer(this); }
public void initialize(Configuration conf) throws IOException, SAXException, AllocationConfigurationException, ParserConfigurationException { rootQueue = new FSParentQueue("root", scheduler, null); queues.put(rootQueue.getName(), rootQueue); // Create the default queue getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME, true); }
@Override public QueueInfo getQueueInfo(String queueName, boolean includeChildQueues, boolean recursive) throws IOException { if (!queueMgr.exists(queueName)) { throw new IOException("queue " + queueName + " does not exist"); } return queueMgr.getQueue(queueName).getQueueInfo(includeChildQueues, recursive); }
private FSQueue getQueue(String name, boolean create, FSQueueType queueType) { name = ensureRootPrefix(name); synchronized (queues) { FSQueue queue = queues.get(name); if (queue == null && create) { // if the queue doesn't exist,create it and return queue = createQueue(name, queueType); // Update steady fair share for all queues if (queue != null) { rootQueue.recomputeSteadyShares(); } } return queue; } }
@Override protected void createDefaultReservationQueue(String planQueueName, Queue queue, String defReservationId) { String defReservationQueueName = getReservationQueueName(planQueueName, defReservationId); if (!fs.getQueueManager().exists(defReservationQueueName)) { fs.getQueueManager().getLeafQueue(defReservationQueueName, true); } }
throws Exception { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); queueManager = new QueueManager(scheduler); queueManager.initialize(conf); queueManager.updateAllocationConfiguration(allocConf); FSLeafQueue q = queueManager.getLeafQueue("root.leaf1", true); assertNotNull("root.leaf1 does not exist", q); assertTrue("root.leaf1 is not empty", queueManager.isEmpty(q)); q = queueManager.getLeafQueue("root.leaf1", false); assertFalse("root.leaf1 is empty", queueManager.isEmpty(q)); queueManager.removePendingIncompatibleQueues(); queueManager.removeEmptyDynamicQueues(); q = queueManager.getLeafQueue("root.leaf1", false); assertNotNull("root.leaf1 has been removed", q); assertFalse("root.leaf1 is empty", queueManager.isEmpty(q)); "a_user", q, activeUsersManager, rmContext); q.addApp(appAttempt, true); queueManager.removeEmptyDynamicQueues(); q = queueManager.getLeafQueue("root.leaf1", false); assertNotNull("root.leaf1 has been removed", q); assertFalse("root.leaf1 is empty", queueManager.isEmpty(q)); q = queueManager.getLeafQueue("root.leaf1", false); assertTrue("root.leaf1 is not empty", queueManager.isEmpty(q));
/** * Test creation of a leaf queue and its parent. */ @Test public void testCreateLeafQueueAndParent() { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); queueManager.updateAllocationConfiguration(allocConf); FSQueue q2 = queueManager.createQueue("root.queue1.queue2", FSQueueType.LEAF); assertNotNull("Parent queue root.queue1 was not created", queueManager.getParentQueue("root.queue1", false)); assertNotNull("Leaf queue root.queue1.queue2 was not created", queueManager.getLeafQueue("root.queue1.queue2", false)); assertEquals("createQueue() returned wrong queue", "root.queue1.queue2", q2.getName()); }
@Test public void testRemovalOfDynamicLeafQueue() { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); queueManager.updateAllocationConfiguration(allocConf); FSQueue q1 = queueManager.getLeafQueue("root.test.childB.dynamic1", true); assertNotNull("Queue root.test.childB.dynamic1 was not created", q1); assertEquals("createQueue() returned wrong queue", "root.test.childB.dynamic1", q1.getName()); assertTrue("root.test.childB.dynamic1 is not a dynamic queue", q1.isDynamic()); // an application is submitted to root.test.childB.dynamic1 notEmptyQueues.add(q1); // root.test.childB.dynamic1 is not empty and should not be removed queueManager.removePendingIncompatibleQueues(); queueManager.removeEmptyDynamicQueues(); q1 = queueManager.getLeafQueue("root.test.childB.dynamic1", false); assertNotNull("Queue root.test.childB.dynamic1 was deleted", q1); // the application finishes, the next removeEmptyDynamicQueues() should // clean root.test.childB.dynamic1 up, but keep its static parent notEmptyQueues.remove(q1); queueManager.removePendingIncompatibleQueues(); queueManager.removeEmptyDynamicQueues(); q1 = queueManager.getLeafQueue("root.test.childB.dynamic1", false); assertNull("Queue root.test.childB.dynamic1 was not deleted", q1); assertNotNull("The static parent of root.test.childB.dynamic1 was deleted", queueManager.getParentQueue("root.test.childB", false)); }
@Test public void testHierarchicalQueuesSimilarParents() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); QueueManager queueManager = scheduler.getQueueManager(); FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child", true); Assert.assertEquals(2, queueManager.getLeafQueues().size()); Assert.assertNotNull(leafQueue); Assert.assertEquals("root.parent.child", leafQueue.getName()); FSLeafQueue leafQueue2 = queueManager.getLeafQueue("parent", true); Assert.assertNull(leafQueue2); Assert.assertEquals(2, queueManager.getLeafQueues().size()); FSLeafQueue leafQueue3 = queueManager.getLeafQueue("parent.child.grandchild", true); Assert.assertNull(leafQueue3); Assert.assertEquals(2, queueManager.getLeafQueues().size()); FSLeafQueue leafQueue4 = queueManager.getLeafQueue("parent.sister", true); Assert.assertNotNull(leafQueue4); Assert.assertEquals("root.parent.sister", leafQueue4.getName()); Assert.assertEquals(3, queueManager.getLeafQueues().size()); }
@Test public void testRemovalOfChildlessParentQueue() { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); queueManager.updateAllocationConfiguration(allocConf); FSParentQueue q1 = queueManager.getParentQueue("root.test.childB", false); assertNotNull("Queue root.test.childB was not created", q1); assertEquals("createQueue() returned wrong queue", "root.test.childB", q1.getName()); assertFalse("root.test.childB is a dynamic queue", q1.isDynamic()); // static queues should not be deleted queueManager.removePendingIncompatibleQueues(); queueManager.removeEmptyDynamicQueues(); q1 = queueManager.getParentQueue("root.test.childB", false); assertNotNull("Queue root.test.childB was deleted", q1); // next we remove root.test.childB from the allocation config allocConf.configuredQueues.get(FSQueueType.PARENT) .remove("root.test.childB"); queueManager.updateAllocationConfiguration(allocConf); queueManager.setQueuesToDynamic(Collections.singleton("root.test.childB")); // the next removeEmptyDynamicQueues() call should clean // root.test.childB up queueManager.removePendingIncompatibleQueues(); queueManager.removeEmptyDynamicQueues(); q1 = queueManager.getParentQueue("root.leaf1", false); assertNull("Queue root.leaf1 was not deleted", q1); }
/** * Test creation of a simple parent queue. */ @Test public void testCreateParentQueue() { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); queueManager.updateAllocationConfiguration(allocConf); FSQueue q1 = queueManager.createQueue("root.queue1", FSQueueType.PARENT); assertNotNull("Parent queue root.queue1 was not created", queueManager.getParentQueue("root.queue1", false)); assertEquals("createQueue() returned wrong queue", "root.queue1", q1.getName()); }
@Test public void testReloadTurnsLeafToParentWithNoLeaf() { AllocationConfiguration allocConf = new AllocationConfiguration(conf); // Create a leaf queue1 allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.queue1"); queueManager.updateAllocationConfiguration(allocConf); assertNotNull(queueManager.getLeafQueue("root.queue1", false)); // Lets say later on admin makes queue1 a parent queue by // specifying "type=parent" in the alloc xml and lets say apps running in // queue1 notEmptyQueues.add(queueManager.getLeafQueue("root.queue1", false)); allocConf = new AllocationConfiguration(conf); allocConf.configuredQueues.get(FSQueueType.PARENT) .add("root.queue1"); // When allocs are reloaded queue1 shouldn't be converter to parent queueManager.updateAllocationConfiguration(allocConf); assertNotNull(queueManager.getLeafQueue("root.queue1", false)); assertNull(queueManager.getParentQueue("root.queue1", false)); // Now lets assume apps completed and there are no apps in queue1 notEmptyQueues.clear(); // We should see queue1 transform from leaf queue to parent queue. queueManager.updateAllocationConfiguration(allocConf); assertNull(queueManager.getLeafQueue("root.queue1", false)); assertNotNull(queueManager.getParentQueue("root.queue1", false)); // this parent should not have any children assertTrue(queueManager.getParentQueue("root.queue1", false) .getChildQueues().isEmpty()); }
/** * Test simple leaf queue creation. */ @Test public void testCreateLeafQueue() { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); queueManager.updateAllocationConfiguration(allocConf); FSQueue q1 = queueManager.createQueue("root.queue1", FSQueueType.LEAF); assertNotNull("Leaf queue root.queue1 was not created", queueManager.getLeafQueue("root.queue1", false)); assertEquals("createQueue() returned wrong queue", "root.queue1", q1.getName()); }
@Test public void testEmptyChildQueues() throws Exception { FairSchedulerConfiguration conf = new FairSchedulerConfiguration(); FairScheduler scheduler = mock(FairScheduler.class); AllocationConfiguration allocConf = new AllocationConfiguration(conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(1, 1)); when(scheduler.getResourceCalculator()).thenReturn( new DefaultResourceCalculator()); SystemClock clock = SystemClock.getInstance(); when(scheduler.getClock()).thenReturn(clock); QueueManager queueManager = new QueueManager(scheduler); queueManager.initialize(conf); FSQueue testQueue = queueManager.getLeafQueue("test", true); FairSchedulerQueueInfo queueInfo = new FairSchedulerQueueInfo(testQueue, scheduler); Collection<FairSchedulerQueueInfo> childQueues = queueInfo.getChildQueues(); Assert.assertNotNull(childQueues); Assert.assertEquals("Child QueueInfo was not empty", 0, childQueues.size()); } }
@Override public List<QueueUserACLInfo> getQueueUserAclInfo() { UserGroupInformation user; try { user = UserGroupInformation.getCurrentUser(); } catch (IOException ioe) { return new ArrayList<QueueUserACLInfo>(); } return queueMgr.getRootQueue().getQueueUserAclInfo(user); }
@Before public void setup() throws Exception { Configuration conf = new Configuration(); clock = new TestFairScheduler.MockClock(); scheduler = mock(FairScheduler.class); when(scheduler.getConf()).thenReturn( new FairSchedulerConfiguration(conf)); when(scheduler.getClock()).thenReturn(clock); AllocationConfiguration allocConf = new AllocationConfiguration( conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); queueManager = new QueueManager(scheduler); queueManager.initialize(conf); queueMaxApps = allocConf.queueMaxApps; userMaxApps = allocConf.userMaxApps; maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler); appNum = 0; rmContext = mock(RMContext.class); when(rmContext.getEpoch()).thenReturn(0L); }
@Override protected Queue getPlanQueue(String planQueueName) { Queue planQueue = fs.getQueueManager().getParentQueue(planQueueName, false); if (planQueue == null) { LOG.error("The queue " + planQueueName + " cannot be found or is not a " + "ParentQueue"); } return planQueue; }
/** * Removes all empty dynamic queues (including empty dynamic parent queues). */ public void removeEmptyDynamicQueues() { synchronized (queues) { Set<FSParentQueue> parentQueuesToCheck = new HashSet<>(); for (FSQueue queue : getQueues()) { if (queue.isDynamic() && queue.getChildQueues().isEmpty()) { boolean removed = removeQueueIfEmpty(queue); if (removed && queue.getParent().isDynamic()) { parentQueuesToCheck.add(queue.getParent()); } } } while (!parentQueuesToCheck.isEmpty()) { FSParentQueue queue = parentQueuesToCheck.iterator().next(); if (queue.getChildQueues().isEmpty()) { removeQueue(queue); if (queue.getParent().isDynamic()) { parentQueuesToCheck.add(queue.getParent()); } } parentQueuesToCheck.remove(queue); } } }
/** * Update the preemption fields for all QueueScheduables, i.e. the times since * each queue last was at its guaranteed share and over its fair share * threshold for each type of task. */ private void updateStarvationStats() { lastPreemptionUpdateTime = clock.getTime(); for (FSLeafQueue sched : queueMgr.getLeafQueues()) { sched.updateStarvationStats(); } }