allocConf.set(new AllocationConfiguration(conf)); QueuePlacementPolicy queuePolicy = allocConf.get().getPlacementPolicy(); if (queuePolicy != null) { requestedQueue = queuePolicy.assignAppToQueue(requestedQueue, userName);
assertEquals(5, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size()); assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); queueConf.getMinResources("root.queueA")); assertEquals(Resources.createResource(2048, 0), queueConf.getMinResources("root.queueB")); assertEquals(Resources.createResource(0), queueConf.getMinResources("root.queueC")); assertEquals(Resources.createResource(0), queueConf.getMinResources("root.queueD")); assertEquals(Resources.createResource(0), queueConf.getMinResources("root.queueE")); assertEquals(15, queueConf.getQueueMaxApps("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); assertEquals(15, queueConf.getQueueMaxApps("root.queueA")); assertEquals(15, queueConf.getQueueMaxApps("root.queueB")); assertEquals(15, queueConf.getQueueMaxApps("root.queueC")); assertEquals(3, queueConf.getQueueMaxApps("root.queueD")); assertEquals(15, queueConf.getQueueMaxApps("root.queueE")); assertEquals(10, queueConf.getUserMaxApps("user1")); assertEquals(5, queueConf.getUserMaxApps("user2")); assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root")); assertEquals(-1, queueConf.getMinSharePreemptionTimeout("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); assertEquals(-1, queueConf.getMinSharePreemptionTimeout("root.queueA"));
assertEquals(6, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size()); assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); queueConf.getMaxResources("root.queueA").getResource()); assertEquals(Resources.createResource(5120, 110), queueConf.getMaxResources("root.queueB").getResource()); assertEquals(Resources.createResource(4096, 100), queueConf.getMaxResources("root.queueC").getResource()); assertEquals(Resources.createResource(4096, 100), queueConf.getMaxResources("root.queueD").getResource()); assertEquals(Resources.createResource(4096, 100), queueConf.getMaxResources("root.queueE").getResource()); assertEquals(Resources.createResource(4096, 100), queueConf.getMaxResources("root.queueF").getResource()); assertEquals(Resources.createResource(4096, 100), queueConf.getMaxResources("root.queueG").getResource()); assertEquals(Resources.createResource(4096, 100), queueConf.getMaxResources("root.queueG.queueH").getResource()); queueConf.getMinResources("root.queueA")); assertEquals(Resources.createResource(2048, 0), queueConf.getMinResources("root.queueB")); assertEquals(Resources.createResource(5120, 0), queueConf.getMinResources("root.queueC")); assertEquals(Resources.createResource(0), queueConf.getMinResources("root.queueD")); assertEquals(Resources.createResource(0), queueConf.getMinResources("root.queueE"));
/** * Initialize a {@link FSQueue} with queue-specific properties and its * metrics. * @param queue the FSQueue needed to be initialized */ public void initFSQueue(FSQueue queue){ // Set queue-specific properties. String name = queue.getName(); queue.setWeights(getQueueWeight(name)); queue.setMinShare(getMinResources(name)); queue.setMaxShare(getMaxResources(name)); queue.setMaxRunningApps(getQueueMaxApps(name)); queue.setMaxAMShare(getQueueMaxAMShare(name)); queue.setMaxChildQueueResource(getMaxChildResources(name)); queue.setMaxContainerAllocation(getQueueMaxContainerAllocation(name)); // Set queue metrics. queue.getMetrics().setMinShare(queue.getMinShare()); queue.getMetrics().setMaxShare(queue.getMaxShare()); queue.getMetrics().setMaxApps(queue.getMaxRunningApps()); queue.getMetrics().setSchedulingPolicy(getSchedulingPolicy(name).getName()); } }
String reservableQueueName = "root.reservable"; String nonreservableQueueName = "root.other"; assertFalse(allocConf.isReservable(nonreservableQueueName)); assertTrue(allocConf.isReservable(reservableQueueName)); Map<FSQueueType, Set<String>> configuredQueues = allocConf.getConfiguredQueues(); assertTrue("reservable queue is expected be to a parent queue", configuredQueues.get(FSQueueType.PARENT).contains(reservableQueueName)); assertTrue(allocConf.getMoveOnExpiry(reservableQueueName)); assertEquals(ReservationSchedulerConfiguration.DEFAULT_RESERVATION_WINDOW, allocConf.getReservationWindow(reservableQueueName)); assertEquals(100, allocConf.getInstantaneousMaxCapacity (reservableQueueName), 0.0001); assertEquals( "DummyAgentName", allocConf.getReservationAgent(reservableQueueName)); assertEquals(100, allocConf.getAverageCapacity(reservableQueueName), 0.001); assertFalse(allocConf.getShowReservationAsQueues(reservableQueueName)); assertEquals("AnyAdmissionPolicy", allocConf.getReservationAdmissionPolicy(reservableQueueName)); assertEquals(ReservationSchedulerConfiguration .DEFAULT_RESERVATION_PLANNER_NAME, allocConf.getReplanner(reservableQueueName)); assertEquals(ReservationSchedulerConfiguration .DEFAULT_RESERVATION_ENFORCEMENT_WINDOW, allocConf.getEnforcementWindow(reservableQueueName));
private void updateConfiguredLeafQueues(QueueManager queueMgr, String... confLeafQueues) { AllocationConfiguration allocConf = new AllocationConfiguration(conf); allocConf.configuredQueues.get(FSQueueType.LEAF).addAll(Sets.newHashSet(confLeafQueues)); queueMgr.updateAllocationConfiguration(allocConf); }
QueuePlacementPolicy policy = allocConf.getPlacementPolicy(); List<QueuePlacementRule> rules = policy.getRules(); assertEquals(1, rules.size()); assertEquals(QueuePlacementRule.Default.class, rules.get(0).getClass()); assertEquals(1, allocConf.getQueueMaxApps("root.queueA")); assertEquals(2, allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .size()); assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .contains("root.queueA")); assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .contains("root.queueB")); policy = allocConf.getPlacementPolicy(); rules = policy.getRules(); assertEquals(3, rules.size()); ((NestedUserQueue) (rules.get(1))).nestedRule.getClass()); assertEquals(QueuePlacementRule.Default.class, rules.get(2).getClass()); assertEquals(3, allocConf.getQueueMaxApps("root.queueB")); assertEquals(1, allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .size()); assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF) .contains("root.queueB"));
/** * Assign reservation to a queue based on the placement policy specified * in the alloc conf */ // Visible for testing String assignToQueue(TLlamaAMReservationRequest request) throws LlamaException { // Default means no queue requested String requestedQueue = (request.isSetQueue()) ? request.getQueue() : YarnConfiguration.DEFAULT_QUEUE_NAME; if (requestedQueue == null) { requestedQueue = YarnConfiguration.DEFAULT_QUEUE_NAME; } String user = request.getUser(); String queue; try { queue = allocConf.get().getPlacementPolicy() .assignAppToQueue(requestedQueue, user); } catch (IOException ex) { throw new LlamaException(ex, ErrorCode.INTERNAL_ERROR); } if (queue == null) { throw new LlamaException( ErrorCode.RESERVATION_USER_TO_QUEUE_MAPPING_NOT_FOUND, user, requestedQueue); } LOG.debug("Reservation from user " + user + " with requested queue " + requestedQueue + " resolved to queue " + queue); return queue; }
/** * Update the min/fair share preemption timeouts and threshold for this queue. */ public void updatePreemptionVariables() { // For min share timeout minSharePreemptionTimeout = scheduler.getAllocationConfiguration() .getMinSharePreemptionTimeout(getName()); if (minSharePreemptionTimeout == -1 && parent != null) { minSharePreemptionTimeout = parent.getMinSharePreemptionTimeout(); } // For fair share timeout fairSharePreemptionTimeout = scheduler.getAllocationConfiguration() .getFairSharePreemptionTimeout(getName()); if (fairSharePreemptionTimeout == -1 && parent != null) { fairSharePreemptionTimeout = parent.getFairSharePreemptionTimeout(); } // For fair share preemption threshold fairSharePreemptionThreshold = scheduler.getAllocationConfiguration() .getFairSharePreemptionThreshold(getName()); if (fairSharePreemptionThreshold < 0 && parent != null) { fairSharePreemptionThreshold = parent.getFairSharePreemptionThreshold(); } }
fractionMemMaxShare = (float)maxResources.getMemory() / clusterResources.getMemory(); maxApps = allocConf.getQueueMaxApps(queueName); if (allocConf.isReservable(queueName) && !allocConf.getShowReservationAsQueues(queueName)) { return;
.getMinSharePreemptionTimeout(getName()); if (minSharePreemptionTimeout == -1 && parent != null) { minSharePreemptionTimeout = parent.getMinSharePreemptionTimeout(); .getFairSharePreemptionTimeout(getName()); if (fairSharePreemptionTimeout == -1 && parent != null) { fairSharePreemptionTimeout = parent.getFairSharePreemptionTimeout(); .getFairSharePreemptionThreshold(getName()); if (fairSharePreemptionThreshold < 0 && parent != null) { fairSharePreemptionThreshold = parent.getFairSharePreemptionThreshold(); } else { preemptable = scheduler.getAllocationConfiguration() .isPreemptable(getName());
/** * Checks whether making the application runnable would exceed any * maxRunningApps limits. */ public boolean canAppBeRunnable(FSQueue queue, String user) { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); Integer userNumRunnable = usersNumRunnableApps.get(user); if (userNumRunnable == null) { userNumRunnable = 0; } if (userNumRunnable >= allocConf.getUserMaxApps(user)) { return false; } // Check queue and all parent queues while (queue != null) { int queueMaxApps = allocConf.getQueueMaxApps(queue.getName()); if (queue.getNumRunnableApps() >= queueMaxApps) { return false; } queue = queue.getParent(); } return true; }
@Override public Resource getMaxShare() { return scheduler.getAllocationConfiguration().getMaxResources(getName()); }
public void updateAllocationConfiguration(AllocationConfiguration queueConf) { for (String name : queueConf.getConfiguredQueues().get(FSQueueType.LEAF)) { if (removeEmptyIncompatibleQueues(name, FSQueueType.LEAF)) { getLeafQueue(name, true); for (String name : queueConf.getConfiguredQueues().get( FSQueueType.PARENT)) { if (removeEmptyIncompatibleQueues(name, FSQueueType.PARENT)) { SchedulingPolicy policy = queueConf.getSchedulingPolicy(queue.getName()); policy.initialize(scheduler.getClusterResource()); queue.setPolicy(policy);
@Test public void testUpdateDemand() { conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); resourceManager = new MockRM(conf); resourceManager.start(); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); scheduler.allocConf = mock(AllocationConfiguration.class); String queueName = "root.queue1"; when(scheduler.allocConf.getMaxResources(queueName)).thenReturn(maxResource); when(scheduler.allocConf.getMinResources(queueName)).thenReturn(Resources.none()); FSLeafQueue schedulable = new FSLeafQueue(queueName, scheduler, null); FSAppAttempt app = mock(FSAppAttempt.class); Mockito.when(app.getDemand()).thenReturn(maxResource); schedulable.addAppSchedulable(app); schedulable.addAppSchedulable(app); schedulable.updateDemand(); assertTrue("Demand is greater than max allowed ", Resources.equals(schedulable.getDemand(), maxResource)); }
private Set<String> getRemovedStaticQueues( AllocationConfiguration queueInfo) { if (queueInfo == null || allocConf == null) { return Collections.emptySet(); } Set<String> removedStaticQueues = new HashSet<>(); for (Set<String> queues : allocConf.getConfiguredQueues().values()) { removedStaticQueues.addAll(queues); } for (Set<String> queues : queueInfo.getConfiguredQueues().values()) { removedStaticQueues.removeAll(queues); } return removedStaticQueues; }
/** * Traverses the queue hierarchy under the given queue to gather all lists * of non-runnable applications. */ private void gatherPossiblyRunnableAppLists(FSQueue queue, List<List<FSAppAttempt>> appLists) { if (queue.getNumRunnableApps() < scheduler.getAllocationConfiguration() .getQueueMaxApps(queue.getName())) { if (queue instanceof FSLeafQueue) { appLists.add( ((FSLeafQueue)queue).getCopyOfNonRunnableAppSchedulables()); } else { for (FSQueue child : queue.getChildQueues()) { gatherPossiblyRunnableAppLists(child, appLists); } } } }
@Override public Resource getMinShare() { return scheduler.getAllocationConfiguration().getMinResources(getName()); }
/** * Check whether this queue can run this application master under the * maxAMShare limit * * @param amResource * @return true if this queue can run */ public boolean canRunAppAM(Resource amResource) { float maxAMShare = scheduler.getAllocationConfiguration().getQueueMaxAMShare(getName()); if (Math.abs(maxAMShare - -1.0f) < 0.0001) { return true; } Resource maxAMResource = Resources.multiply(getFairShare(), maxAMShare); Resource ifRunAMResource = Resources.add(amResourceUsage, amResource); return !policy .checkIfAMResourceUsageOverLimit(ifRunAMResource, maxAMResource); }
assertEquals(5, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size()); assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); queueConf.getMinResources("root.queueA")); assertEquals(Resources.createResource(2048, 0), queueConf.getMinResources("root.queueB")); assertEquals(Resources.createResource(0), queueConf.getMinResources("root.queueC")); assertEquals(Resources.createResource(0), queueConf.getMinResources("root.queueD")); assertEquals(Resources.createResource(0), queueConf.getMinResources("root.queueE")); assertEquals(15, queueConf.getQueueMaxApps("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); assertEquals(15, queueConf.getQueueMaxApps("root.queueA")); assertEquals(15, queueConf.getQueueMaxApps("root.queueB")); assertEquals(15, queueConf.getQueueMaxApps("root.queueC")); assertEquals(3, queueConf.getQueueMaxApps("root.queueD")); assertEquals(15, queueConf.getQueueMaxApps("root.queueE")); assertEquals(10, queueConf.getUserMaxApps("user1")); assertEquals(5, queueConf.getUserMaxApps("user2")); assertEquals(" ", queueConf.getQueueAcl("root.queueA", QueueACL.ADMINISTER_QUEUE).getAclString()); assertEquals(" ", queueConf.getQueueAcl("root.queueA", QueueACL.SUBMIT_APPLICATIONS).getAclString());