@Override protected List<? extends Queue> getChildReservationQueues(Queue queue) { FSQueue planQueue = (FSQueue)queue; List<FSQueue> childQueues = planQueue.getChildQueues(); return childQueues; }
@Override protected List<? extends Queue> getChildReservationQueues(Queue queue) { FSQueue planQueue = (FSQueue)queue; List<FSQueue> childQueues = planQueue.getChildQueues(); return childQueues; }
@Override protected List<? extends Queue> getChildReservationQueues(Queue queue) { FSQueue planQueue = (FSQueue)queue; List<FSQueue> childQueues = planQueue.getChildQueues(); return childQueues; }
/** * Remove a queue and all its descendents. */ private void removeQueue(FSQueue queue) { if (queue instanceof FSLeafQueue) { leafQueues.remove(queue); } else { List<FSQueue> childQueues = queue.getChildQueues(); while (!childQueues.isEmpty()) { removeQueue(childQueues.get(0)); } } queues.remove(queue.getName()); queue.getParent().getChildQueues().remove(queue); }
/** * Remove a queue and all its descendents. */ private void removeQueue(FSQueue queue) { if (queue instanceof FSLeafQueue) { leafQueues.remove(queue); } else { List<FSQueue> childQueues = queue.getChildQueues(); while (!childQueues.isEmpty()) { removeQueue(childQueues.get(0)); } } queues.remove(queue.getName()); queue.getParent().getChildQueues().remove(queue); }
/** * Remove a queue and all its descendents. */ private void removeQueue(FSQueue queue) { synchronized (queues) { if (queue instanceof FSLeafQueue) { leafQueues.remove(queue); } else { for (FSQueue childQueue:queue.getChildQueues()) { removeQueue(childQueue); } } queues.remove(queue.getName()); FSParentQueue parent = queue.getParent(); parent.removeChildQueue(queue); } }
/** * Returns true if there are no applications, running or not, in the given * queue or any of its descendents. */ protected boolean isEmpty(FSQueue queue) { if (queue instanceof FSLeafQueue) { FSLeafQueue leafQueue = (FSLeafQueue)queue; return queue.getNumRunnableApps() == 0 && leafQueue.getNumNonRunnableApps() == 0; } else { for (FSQueue child : queue.getChildQueues()) { if (!isEmpty(child)) { return false; } } return true; } }
/** * Traverses the queue hierarchy under the given queue to gather all lists * of non-runnable applications. */ private void gatherPossiblyRunnableAppLists(FSQueue queue, List<List<FSAppAttempt>> appLists) { if (queue.getNumRunnableApps() < queue.getMaxRunningApps()) { if (queue instanceof FSLeafQueue) { appLists.add( ((FSLeafQueue)queue).getCopyOfNonRunnableAppSchedulables()); } else { for (FSQueue child : queue.getChildQueues()) { gatherPossiblyRunnableAppLists(child, appLists); } } } }
/** * Returns true if there are no applications, running or not, in the given * queue or any of its descendents. */ protected boolean isEmpty(FSQueue queue) { if (queue instanceof FSLeafQueue) { FSLeafQueue leafQueue = (FSLeafQueue)queue; return queue.getNumRunnableApps() == 0 && leafQueue.getNumNonRunnableApps() == 0; } else { for (FSQueue child : queue.getChildQueues()) { if (!isEmpty(child)) { return false; } } return true; } }
/** * Removes all empty dynamic queues (including empty dynamic parent queues). */ public void removeEmptyDynamicQueues() { synchronized (queues) { Set<FSParentQueue> parentQueuesToCheck = new HashSet<>(); for (FSQueue queue : getQueues()) { if (queue.isDynamic() && queue.getChildQueues().isEmpty()) { boolean removed = removeQueueIfEmpty(queue); if (removed && queue.getParent().isDynamic()) { parentQueuesToCheck.add(queue.getParent()); } } } while (!parentQueuesToCheck.isEmpty()) { FSParentQueue queue = parentQueuesToCheck.iterator().next(); if (queue.getChildQueues().isEmpty()) { removeQueue(queue); if (queue.getParent().isDynamic()) { parentQueuesToCheck.add(queue.getParent()); } } parentQueuesToCheck.remove(queue); } } }
/** * Initialize a queue by setting its queue-specific properties and its * metrics. This method is invoked when creating a new queue or reloading * the allocation file. * This method does not set policies for queues when reloading the allocation * file since we need to either set all new policies or nothing, which is * handled by method {@link #verifyAndSetPolicyFromConf}. * * @param recursive whether child queues should be reinitialized recursively */ public final void reinit(boolean recursive) { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); allocConf.initFSQueue(this); updatePreemptionVariables(); if (recursive) { for (FSQueue child : getChildQueues()) { child.reinit(recursive); } } }
/** * Returns true if there are no applications, running or not, in the given * queue or any of its descendents. */ protected boolean isEmpty(FSQueue queue) { if (queue instanceof FSLeafQueue) { FSLeafQueue leafQueue = (FSLeafQueue)queue; return queue.getNumRunnableApps() == 0 && leafQueue.getNumNonRunnableApps() == 0 && leafQueue.getNumAssignedApps() == 0; } else { for (FSQueue child : queue.getChildQueues()) { if (!isEmpty(child)) { return false; } } return true; } }
/** * Traverses the queue hierarchy under the given queue to gather all lists * of non-runnable applications. */ private void gatherPossiblyRunnableAppLists(FSQueue queue, List<List<FSAppAttempt>> appLists) { if (queue.getNumRunnableApps() < scheduler.getAllocationConfiguration() .getQueueMaxApps(queue.getName())) { if (queue instanceof FSLeafQueue) { appLists.add( ((FSLeafQueue)queue).getCopyOfNonRunnableAppSchedulables()); } else { for (FSQueue child : queue.getChildQueues()) { gatherPossiblyRunnableAppLists(child, appLists); } } } }
protected FairSchedulerQueueInfoList getChildQueues(FSQueue queue, FairScheduler scheduler) { // Return null to omit 'childQueues' field from the return value of // REST API if it is empty. We omit the field to keep the consistency // with CapacitySchedulerQueueInfo, which omits 'queues' field if empty. Collection<FSQueue> children = queue.getChildQueues(); if (children.isEmpty()) { return null; } FairSchedulerQueueInfoList list = new FairSchedulerQueueInfoList(); for (FSQueue child : children) { if (child instanceof FSLeafQueue) { list.addToQueueInfoList( new FairSchedulerLeafQueueInfo((FSLeafQueue) child, scheduler)); } else { list.addToQueueInfoList( new FairSchedulerQueueInfo(child, scheduler)); } } return list; }
/** * Traverses the queue hierarchy under the given queue to gather all lists * of non-runnable applications. */ private void gatherPossiblyRunnableAppLists(FSQueue queue, List<List<FSAppAttempt>> appLists) { if (queue.getNumRunnableApps() < scheduler.getAllocationConfiguration() .getQueueMaxApps(queue.getName())) { if (queue instanceof FSLeafQueue) { appLists.add( ((FSLeafQueue)queue).getCopyOfNonRunnableAppSchedulables()); } else { for (FSQueue child : queue.getChildQueues()) { gatherPossiblyRunnableAppLists(child, appLists); } } } }
@Test public void testLowestCommonAncestorDeeperHierarchy() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); FSQueue aQueue = mock(FSLeafQueue.class); FSQueue bQueue = mock(FSLeafQueue.class); FSQueue a1Queue = mock(FSLeafQueue.class); FSQueue b1Queue = mock(FSLeafQueue.class); when(a1Queue.getName()).thenReturn("root.queue1.a.a1"); when(b1Queue.getName()).thenReturn("root.queue1.b.b1"); when(aQueue.getChildQueues()).thenReturn(Arrays.asList(a1Queue)); when(bQueue.getChildQueues()).thenReturn(Arrays.asList(b1Queue)); QueueManager queueManager = scheduler.getQueueManager(); FSParentQueue queue1 = queueManager.getParentQueue("queue1", true); queue1.addChildQueue(aQueue); queue1.addChildQueue(bQueue); FSQueue ancestorQueue = scheduler.findLowestCommonAncestorQueue(a1Queue, b1Queue); assertEquals(ancestorQueue, queue1); }
@Test public void testLowestCommonAncestorDeeperHierarchy() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); FSQueue aQueue = mock(FSLeafQueue.class); FSQueue bQueue = mock(FSLeafQueue.class); FSQueue a1Queue = mock(FSLeafQueue.class); FSQueue b1Queue = mock(FSLeafQueue.class); when(a1Queue.getName()).thenReturn("root.queue1.a.a1"); when(b1Queue.getName()).thenReturn("root.queue1.b.b1"); when(aQueue.getChildQueues()).thenReturn(Arrays.asList(a1Queue)); when(bQueue.getChildQueues()).thenReturn(Arrays.asList(b1Queue)); QueueManager queueManager = scheduler.getQueueManager(); FSParentQueue queue1 = queueManager.getParentQueue("queue1", true); queue1.addChildQueue(aQueue); queue1.addChildQueue(bQueue); FSQueue ancestorQueue = scheduler.findLowestCommonAncestorQueue(a1Queue, b1Queue); assertEquals(ancestorQueue, queue1); }
/** * Recursively check policies for queues in pre-order. Get queue policies * from the allocation file instead of properties of {@link FSQueue} objects. * Set the policy for current queue if there is no policy violation for its * children. This method is invoked while reloading the allocation file. * * @param queueConf allocation configuration * @return true if no policy violation and successfully set polices * for queues; false otherwise */ public boolean verifyAndSetPolicyFromConf(AllocationConfiguration queueConf) { SchedulingPolicy queuePolicy = queueConf.getSchedulingPolicy(getName()); for (FSQueue child : getChildQueues()) { if (!queuePolicy.isChildPolicyAllowed( queueConf.getSchedulingPolicy(child.getName()))) { return false; } boolean success = child.verifyAndSetPolicyFromConf(queueConf); if (!success) { return false; } } // Set the policy if no policy violation for all children setPolicy(queuePolicy); return true; }
@Override public QueueInfo getQueueInfo(boolean includeChildQueues, boolean recursive) { QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class); queueInfo.setQueueName(getQueueName()); if (scheduler.getClusterResource().getMemory() == 0) { queueInfo.setCapacity(0.0f); } else { queueInfo.setCapacity((float) getFairShare().getMemory() / scheduler.getClusterResource().getMemory()); } if (getFairShare().getMemory() == 0) { queueInfo.setCurrentCapacity(0.0f); } else { queueInfo.setCurrentCapacity((float) getResourceUsage().getMemory() / getFairShare().getMemory()); } ArrayList<QueueInfo> childQueueInfos = new ArrayList<QueueInfo>(); if (includeChildQueues) { Collection<FSQueue> childQueues = getChildQueues(); for (FSQueue child : childQueues) { childQueueInfos.add(child.getQueueInfo(recursive, recursive)); } } queueInfo.setChildQueues(childQueueInfos); queueInfo.setQueueState(QueueState.RUNNING); return queueInfo; }
@Override public QueueInfo getQueueInfo(boolean includeChildQueues, boolean recursive) { QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class); queueInfo.setQueueName(getQueueName()); if (scheduler.getClusterResource().getMemorySize() == 0) { queueInfo.setCapacity(0.0f); } else { queueInfo.setCapacity((float) getFairShare().getMemorySize() / scheduler.getClusterResource().getMemorySize()); } if (getFairShare().getMemorySize() == 0) { queueInfo.setCurrentCapacity(0.0f); } else { queueInfo.setCurrentCapacity((float) getResourceUsage().getMemorySize() / getFairShare().getMemorySize()); } ArrayList<QueueInfo> childQueueInfos = new ArrayList<QueueInfo>(); if (includeChildQueues) { Collection<FSQueue> childQueues = getChildQueues(); for (FSQueue child : childQueues) { childQueueInfos.add(child.getQueueInfo(recursive, recursive)); } } queueInfo.setChildQueues(childQueueInfos); queueInfo.setQueueState(QueueState.RUNNING); queueInfo.setQueueStatistics(getQueueStatistics()); return queueInfo; }