private void verifyNoPreemption() throws InterruptedException { // Sleep long enough to ensure not even one container is preempted. for (int i = 0; i < 100; i++) { if (greedyApp.getLiveContainers().size() != 8) { break; } Thread.sleep(10); } assertEquals(8, greedyApp.getLiveContainers().size()); }
/** * Preempt a running container according to the priority */ @Override public RMContainer preemptContainer() { if (LOG.isDebugEnabled()) { LOG.debug("App " + getName() + " is going to preempt a running " + "container"); } RMContainer toBePreempted = null; for (RMContainer container : getLiveContainers()) { if (!getPreemptionContainers().contains(container) && (toBePreempted == null || comparator.compare(toBePreempted, container) > 0)) { toBePreempted = container; } } return toBePreempted; } }
/** * Preempt a running container according to the priority */ @Override public RMContainer preemptContainer() { if (LOG.isDebugEnabled()) { LOG.debug("App " + getName() + " is going to preempt a running " + "container"); } RMContainer toBePreempted = null; for (RMContainer container : getLiveContainers()) { if (!getPreemptionContainers().contains(container) && (toBePreempted == null || comparator.compare(toBePreempted, container) > 0)) { toBePreempted = container; } } return toBePreempted; } }
if (greedyApp.getLiveContainers().size() == numGreedyAppContainers) { break; numGreedyAppContainers, greedyApp.getLiveContainers().size()); numStarvedAppContainers, starvingApp.getLiveContainers().size());
@Test public void testPreemptionSelectNonAMContainer() throws Exception { takeAllResources("root.preemptable.child-1"); setNumAMContainersPerNode(2); preemptHalfResources("root.preemptable.child-2"); verifyPreemption(2, 4); ArrayList<RMContainer> containers = (ArrayList<RMContainer>) starvingApp.getLiveContainers(); String host0 = containers.get(0).getNodeId().getHost(); String host1 = containers.get(1).getNodeId().getHost(); // Each node provides two and only two non-AM containers to be preempted, so // the preemption happens on both nodes. assertTrue("Preempted containers should come from two different " + "nodes.", !host0.equals(host1)); }
assertEquals(1, app1.getLiveContainers().size()); assertEquals(0, app2.getLiveContainers().size()); assertEquals(2, app1.getLiveContainers().size()); assertEquals(0, app2.getLiveContainers().size()); assertEquals(2, app1.getLiveContainers().size()); assertEquals(1, app2.getLiveContainers().size());
assertEquals(1, app1.getLiveContainers().size()); assertEquals(0, app2.getLiveContainers().size()); assertEquals(2, app1.getLiveContainers().size()); assertEquals(0, app2.getLiveContainers().size()); assertEquals(2, app1.getLiveContainers().size()); assertEquals(1, app2.getLiveContainers().size());
Assert.assertEquals(1, app1.getLiveContainers().size()); Assert.assertEquals(0, app2.getLiveContainers().size()); Assert.assertEquals(1, app1.getLiveContainers().size()); Assert.assertEquals(1, app2.getLiveContainers().size()); Assert.assertEquals(2, app1.getLiveContainers().size()); Assert.assertEquals(1, app2.getLiveContainers().size());
Assert.assertEquals(1, app1.getLiveContainers().size()); Assert.assertEquals(0, app2.getLiveContainers().size()); Assert.assertEquals(1, app1.getLiveContainers().size()); Assert.assertEquals(1, app2.getLiveContainers().size()); Assert.assertEquals(2, app1.getLiveContainers().size()); Assert.assertEquals(1, app2.getLiveContainers().size());
/** * Submit an application to a given queue and take over the entire cluster. * * @param queueName queue name */ private void takeAllResources(String queueName) { // Create an app that takes up all the resources on the cluster ApplicationAttemptId appAttemptId = createSchedulingRequest(GB, 1, queueName, "default", NODE_CAPACITY_MULTIPLE * rmNodes.size()); greedyApp = scheduler.getSchedulerApp(appAttemptId); scheduler.update(); sendEnoughNodeUpdatesToAssignFully(); assertEquals(8, greedyApp.getLiveContainers().size()); // Verify preemptable for queue and app attempt assertTrue( scheduler.getQueueManager().getQueue(queueName).isPreemptable() == greedyApp.isPreemptable()); }
@Test(timeout = 3000) public void testMaxAssignWithZeroMemoryContainers() throws Exception { conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(16384, 16), 0, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); scheduler.handle(nodeEvent); ApplicationAttemptId attId = createSchedulingRequest(0, 1, "root.default", "user", 8); FSAppAttempt app = scheduler.getSchedulerApp(attId); // set maxAssign to 2: only 2 containers should be allocated scheduler.maxAssign = 2; scheduler.update(); scheduler.handle(updateEvent); assertEquals("Incorrect number of containers allocated", 2, app .getLiveContainers().size()); // set maxAssign to -1: all remaining containers should be allocated scheduler.maxAssign = -1; scheduler.update(); scheduler.handle(updateEvent); assertEquals("Incorrect number of containers allocated", 8, app .getLiveContainers().size()); }
scheduler.handle(updateEvent); assertEquals("Incorrect number of containers allocated", 2, app .getLiveContainers().size()); scheduler.handle(updateEvent); assertEquals("Incorrect number of containers allocated", 8, app .getLiveContainers().size());
@Test public void testNoMoreCpuOnNode() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); ApplicationAttemptId attId = createSchedulingRequest(1024, 1, "default", "user1", 2); FSAppAttempt app = scheduler.getSchedulerApp(attId); scheduler.update(); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); }
@Test public void testNoMoreCpuOnNode() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); ApplicationAttemptId attId = createSchedulingRequest(1024, 1, "default", "user1", 2); FSAppAttempt app = scheduler.getSchedulerApp(attId); scheduler.update(); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); }
@Test(timeout = 3000) public void testFixedMaxAssign() throws Exception { conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true); conf.setBoolean(FairSchedulerConfiguration.DYNAMIC_MAX_ASSIGN, false); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(16384, 16), 0, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); scheduler.handle(nodeEvent); ApplicationAttemptId attId = createSchedulingRequest(1024, "root.default", "user", 8); FSAppAttempt app = scheduler.getSchedulerApp(attId); // set maxAssign to 2: only 2 containers should be allocated scheduler.maxAssign = 2; scheduler.update(); scheduler.handle(updateEvent); assertEquals("Incorrect number of containers allocated", 2, app .getLiveContainers().size()); // set maxAssign to -1: all remaining containers should be allocated scheduler.maxAssign = -1; scheduler.update(); scheduler.handle(updateEvent); assertEquals("Incorrect number of containers allocated", 8, app .getLiveContainers().size()); }
@Test(timeout = 3000) public void testMaxAssign() throws Exception { conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(16384, 16), 0, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); scheduler.handle(nodeEvent); ApplicationAttemptId attId = createSchedulingRequest(1024, "root.default", "user", 8); FSAppAttempt app = scheduler.getSchedulerApp(attId); // set maxAssign to 2: only 2 containers should be allocated scheduler.maxAssign = 2; scheduler.update(); scheduler.handle(updateEvent); assertEquals("Incorrect number of containers allocated", 2, app .getLiveContainers().size()); // set maxAssign to -1: all remaining containers should be allocated scheduler.maxAssign = -1; scheduler.update(); scheduler.handle(updateEvent); assertEquals("Incorrect number of containers allocated", 8, app .getLiveContainers().size()); }
Assert.assertEquals(1, app1.getLiveContainers().size()); scheduler.handle(updateEvent); Assert.assertEquals(1, app3.getLiveContainers().size()); scheduler.handle(updateEvent); Assert.assertEquals(2, app3.getLiveContainers().size()); scheduler.handle(updateEvent); Assert.assertEquals(1, app2.getLiveContainers().size());
for (int i = 0; i < 10; i++) { scheduler.handle(node2UpdateEvent); assertEquals(0, app.getLiveContainers().size()); assertEquals(0, app.getReservedContainers().size()); assertEquals(1, app.getLiveContainers().size());
for (int i = 0; i < 10; i++) { scheduler.handle(node2UpdateEvent); assertEquals(0, app.getLiveContainers().size()); assertEquals(0, app.getReservedContainers().size()); assertEquals(1, app.getLiveContainers().size());
scheduler.handle(nodeUpdateEvent); assertEquals("App assigned a container on the wrong node", 0, app.getLiveContainers().size()); scheduler.handle(nodeUpdateEvent); assertEquals("App reserved a container on the wrong node",