private void updateDemandForApp(FSAppAttempt sched, Resource maxRes) { sched.updateDemand(); Resource toAdd = sched.getDemand(); if (LOG.isDebugEnabled()) { LOG.debug("Counting resource from " + sched.getName() + " " + toAdd + "; Total resource consumption for " + getName() + " now " + demand); } demand = Resources.add(demand, toAdd); demand = Resources.componentwiseMin(demand, maxRes); }
/** Allows setting weight for a dynamically created queue * Currently only used for reservation based queues * @param weight queue weight */ public void setWeights(float weight) { scheduler.getAllocationConfiguration().setQueueWeight(getName(), new ResourceWeights(weight)); }
/** Allows setting weight for a dynamically created queue * Currently only used for reservation based queues * @param weight queue weight */ public void setWeights(float weight) { scheduler.getAllocationConfiguration().setQueueWeight(getName(), new ResourceWeights(weight)); }
private void updateDemandForApp(FSAppAttempt sched, Resource maxRes) { sched.updateDemand(); Resource toAdd = sched.getDemand(); if (LOG.isDebugEnabled()) { LOG.debug("Counting resource from " + sched.getName() + " " + toAdd + "; Total resource consumption for " + getName() + " now " + demand); } demand = Resources.add(demand, toAdd); demand = Resources.componentwiseMin(demand, maxRes); }
@Override public Resource assignContainer(FSSchedulerNode node) { Resource assigned = none(); if (LOG.isDebugEnabled()) { LOG.debug("Node " + node.getNodeName() + " offered to queue: " + getName() + " fairShare: " + getFairShare()); } if (!assignContainerPreCheck(node)) { return assigned; } for (FSAppAttempt sched : fetchAppsWithDemand(true)) { if (SchedulerAppUtils.isPlaceBlacklisted(sched, node, LOG)) { continue; } assigned = sched.assignContainer(node); if (!assigned.equals(none())) { if (LOG.isDebugEnabled()) { LOG.debug("Assigned container in queue:" + getName() + " " + "container:" + assigned); } break; } } return assigned; }
/** * Check whether this queue can run this application master under the * maxAMShare limit * * @param amResource * @return true if this queue can run */ public boolean canRunAppAM(Resource amResource) { float maxAMShare = scheduler.getAllocationConfiguration().getQueueMaxAMShare(getName()); if (Math.abs(maxAMShare - -1.0f) < 0.0001) { return true; } Resource maxAMResource = Resources.multiply(getFairShare(), maxAMShare); Resource ifRunAMResource = Resources.add(amResourceUsage, amResource); return !policy .checkIfAMResourceUsageOverLimit(ifRunAMResource, maxAMResource); }
/** * Check whether this queue can run this application master under the * maxAMShare limit * * @param amResource * @return true if this queue can run */ public boolean canRunAppAM(Resource amResource) { float maxAMShare = scheduler.getAllocationConfiguration().getQueueMaxAMShare(getName()); if (Math.abs(maxAMShare - -1.0f) < 0.0001) { return true; } Resource maxAMResource = Resources.multiply(getFairShare(), maxAMShare); Resource ifRunAMResource = Resources.add(amResourceUsage, amResource); return !policy .checkIfAMResourceUsageOverLimit(ifRunAMResource, maxAMResource); }
/** * Verify whether steady fair shares for all leaf queues still follow * their weight, not related to active/inactive status. * * @param leafQueues * @param nodeCapacity */ private void verifySteadyFairShareMemory(Collection<FSLeafQueue> leafQueues, int nodeCapacity) { for (FSLeafQueue leaf : leafQueues) { if (leaf.getName().startsWith("root.parentA")) { assertEquals(0.2, (double) leaf.getSteadyFairShare().getMemorySize() / nodeCapacity, 0.001); } else if (leaf.getName().startsWith("root.parentB")) { assertEquals(0.05, (double) leaf.getSteadyFairShare().getMemorySize() / nodeCapacity, 0.001); } } } }
@Test public void testLowestCommonAncestorRootParent() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); FSLeafQueue aQueue = mock(FSLeafQueue.class); FSLeafQueue bQueue = mock(FSLeafQueue.class); when(aQueue.getName()).thenReturn("root.a"); when(bQueue.getName()).thenReturn("root.b"); QueueManager queueManager = scheduler.getQueueManager(); FSParentQueue queue1 = queueManager.getParentQueue("root", false); queue1.addChildQueue(aQueue); queue1.addChildQueue(bQueue); FSQueue ancestorQueue = scheduler.findLowestCommonAncestorQueue(aQueue, bQueue); assertEquals(ancestorQueue, queue1); }
/** * Verify whether steady fair shares for all leaf queues still follow * their weight, not related to active/inactive status. * * @param leafQueues * @param nodeCapacity */ private void verifySteadyFairShareMemory(Collection<FSLeafQueue> leafQueues, int nodeCapacity) { for (FSLeafQueue leaf : leafQueues) { if (leaf.getName().startsWith("root.parentA")) { assertEquals(0.2, (double) leaf.getSteadyFairShare().getMemory() / nodeCapacity, 0.001); } else if (leaf.getName().startsWith("root.parentB")) { assertEquals(0.05, (double) leaf.getSteadyFairShare().getMemory() / nodeCapacity, 0.001); } } } }
@Test public void testLowestCommonAncestorForNonRootParent() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); FSLeafQueue aQueue = mock(FSLeafQueue.class); FSLeafQueue bQueue = mock(FSLeafQueue.class); when(aQueue.getName()).thenReturn("root.queue1.a"); when(bQueue.getName()).thenReturn("root.queue1.b"); QueueManager queueManager = scheduler.getQueueManager(); FSParentQueue queue1 = queueManager.getParentQueue("queue1", true); queue1.addChildQueue(aQueue); queue1.addChildQueue(bQueue); FSQueue ancestorQueue = scheduler.findLowestCommonAncestorQueue(aQueue, bQueue); assertEquals(ancestorQueue, queue1); }
@Test public void testLowestCommonAncestorRootParent() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); FSLeafQueue aQueue = mock(FSLeafQueue.class); FSLeafQueue bQueue = mock(FSLeafQueue.class); when(aQueue.getName()).thenReturn("root.a"); when(bQueue.getName()).thenReturn("root.b"); QueueManager queueManager = scheduler.getQueueManager(); FSParentQueue queue1 = queueManager.getParentQueue("root", false); queue1.addChildQueue(aQueue); queue1.addChildQueue(bQueue); FSQueue ancestorQueue = scheduler.findLowestCommonAncestorQueue(aQueue, bQueue); assertEquals(ancestorQueue, queue1); }
@Test public void testLowestCommonAncestorForNonRootParent() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); FSLeafQueue aQueue = mock(FSLeafQueue.class); FSLeafQueue bQueue = mock(FSLeafQueue.class); when(aQueue.getName()).thenReturn("root.queue1.a"); when(bQueue.getName()).thenReturn("root.queue1.b"); QueueManager queueManager = scheduler.getQueueManager(); FSParentQueue queue1 = queueManager.getParentQueue("queue1", true); queue1.addChildQueue(aQueue); queue1.addChildQueue(bQueue); FSQueue ancestorQueue = scheduler.findLowestCommonAncestorQueue(aQueue, bQueue); assertEquals(ancestorQueue, queue1); }
@Override protected void dumpStateInternal(StringBuilder sb) { sb.append("{Name: " + getName() + ", Weight: " + weights + ", Policy: " + policy.getName() + ", FairShare: " + getFairShare() + ", SteadyFairShare: " + getSteadyFairShare() + ", MaxShare: " + getMaxShare() + ", MinShare: " + minShare + ", ResourceUsage: " + getResourceUsage() + ", Demand: " + getDemand() + ", Runnable: " + getNumRunnableApps() + ", NumPendingApps: " + getNumPendingApps() + ", NonRunnable: " + getNumNonRunnableApps() + ", MaxAMShare: " + maxAMShare + ", MaxAMResource: " + computeMaxAMResource() + ", AMResourceUsage: " + getAmResourceUsage() + ", LastTimeAtMinShare: " + lastTimeAtMinShare + "}"); }
@Test public void testFairShareNoAppsRunning() throws IOException { int nodeCapacity = 16 * 1024; createClusterWithQueuesAndOneNode(nodeCapacity, "fair"); scheduler.update(); // No apps are running in the cluster,verify if fair share is zero // for all queues under parentA and parentB. Collection<FSLeafQueue> leafQueues = scheduler.getQueueManager() .getLeafQueues(); for (FSLeafQueue leaf : leafQueues) { if (leaf.getName().startsWith("root.parentA")) { assertEquals(0, (double) leaf.getFairShare().getMemory() / nodeCapacity, 0); } else if (leaf.getName().startsWith("root.parentB")) { assertEquals(0, (double) leaf.getFairShare().getMemory() / nodeCapacity, 0); } } verifySteadyFairShareMemory(leafQueues, nodeCapacity); }
@Test public void testFairShareNoAppsRunning() throws IOException { int nodeCapacity = 16 * 1024; createClusterWithQueuesAndOneNode(nodeCapacity, "fair"); scheduler.update(); // No apps are running in the cluster,verify if fair share is zero // for all queues under parentA and parentB. Collection<FSLeafQueue> leafQueues = scheduler.getQueueManager() .getLeafQueues(); for (FSLeafQueue leaf : leafQueues) { if (leaf.getName().startsWith("root.parentA")) { assertEquals(0, (double) leaf.getFairShare().getMemorySize() / nodeCapacity, 0); } else if (leaf.getName().startsWith("root.parentB")) { assertEquals(0, (double) leaf.getFairShare().getMemorySize() / nodeCapacity, 0); } } verifySteadyFairShareMemory(leafQueues, nodeCapacity); }
@Test public void testHierarchicalQueuesSimilarParents() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); QueueManager queueManager = scheduler.getQueueManager(); FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child", true); Assert.assertEquals(2, queueManager.getLeafQueues().size()); Assert.assertNotNull(leafQueue); Assert.assertEquals("root.parent.child", leafQueue.getName()); FSLeafQueue leafQueue2 = queueManager.getLeafQueue("parent", true); Assert.assertNull(leafQueue2); Assert.assertEquals(2, queueManager.getLeafQueues().size()); FSLeafQueue leafQueue3 = queueManager.getLeafQueue("parent.child.grandchild", true); Assert.assertNull(leafQueue3); Assert.assertEquals(2, queueManager.getLeafQueues().size()); FSLeafQueue leafQueue4 = queueManager.getLeafQueue("parent.sister", true); Assert.assertNotNull(leafQueue4); Assert.assertEquals("root.parent.sister", leafQueue4.getName()); Assert.assertEquals(3, queueManager.getLeafQueues().size()); }
@Test public void testAssignToQueue() throws Exception { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW); FSLeafQueue queue1 = scheduler.assignToQueue(rmApp1, "default", "asterix"); FSLeafQueue queue2 = scheduler.assignToQueue(rmApp2, "notdefault", "obelix"); // assert FSLeafQueue's name is the correct name is the one set in the RMApp assertEquals(rmApp1.getQueue(), queue1.getName()); assertEquals("root.asterix", rmApp1.getQueue()); assertEquals(rmApp2.getQueue(), queue2.getName()); assertEquals("root.notdefault", rmApp2.getQueue()); }
@Test public void testHierarchicalQueuesSimilarParents() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); QueueManager queueManager = scheduler.getQueueManager(); FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child", true); Assert.assertEquals(2, queueManager.getLeafQueues().size()); Assert.assertNotNull(leafQueue); Assert.assertEquals("root.parent.child", leafQueue.getName()); FSLeafQueue leafQueue2 = queueManager.getLeafQueue("parent", true); Assert.assertNull(leafQueue2); Assert.assertEquals(2, queueManager.getLeafQueues().size()); FSLeafQueue leafQueue3 = queueManager.getLeafQueue("parent.child.grandchild", true); Assert.assertNull(leafQueue3); Assert.assertEquals(2, queueManager.getLeafQueues().size()); FSLeafQueue leafQueue4 = queueManager.getLeafQueue("parent.sister", true); Assert.assertNotNull(leafQueue4); Assert.assertEquals("root.parent.sister", leafQueue4.getName()); Assert.assertEquals(3, queueManager.getLeafQueues().size()); }
@Test public void testAssignToQueue() throws Exception { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW); FSLeafQueue queue1 = scheduler.assignToQueue(rmApp1, "default", "asterix"); FSLeafQueue queue2 = scheduler.assignToQueue(rmApp2, "notdefault", "obelix"); // assert FSLeafQueue's name is the correct name is the one set in the RMApp assertEquals(rmApp1.getQueue(), queue1.getName()); assertEquals("root.asterix", rmApp1.getQueue()); assertEquals(rmApp2.getQueue(), queue2.getName()); assertEquals("root.notdefault", rmApp2.getQueue()); }