/** * Get the fair share assigned to the appAttemptId. * @param appAttemptId * @return The fair share assigned to the appAttemptId, * <code>FairSchedulerInfo#INVALID_FAIR_SHARE</code> if the scheduler does * not know about this application attempt. */ public long getAppFairShare(ApplicationAttemptId appAttemptId) { FSAppAttempt fsAppAttempt = scheduler.getSchedulerApp(appAttemptId); return fsAppAttempt == null ? INVALID_FAIR_SHARE : fsAppAttempt.getFairShare().getMemorySize(); }
/** * Get the fair share assigned to the appAttemptId. * @param appAttemptId * @return The fair share assigned to the appAttemptId, * <code>FairSchedulerInfo#INVALID_FAIR_SHARE</code> if the scheduler does * not know about this application attempt. */ public int getAppFairShare(ApplicationAttemptId appAttemptId) { FSAppAttempt fsAppAttempt = scheduler.getSchedulerApp(appAttemptId); return fsAppAttempt == null ? INVALID_FAIR_SHARE : fsAppAttempt.getFairShare().getMemory(); }
/** * Get the fair share assigned to the appAttemptId. * @param appAttemptId * @return The fair share assigned to the appAttemptId, * <code>FairSchedulerInfo#INVALID_FAIR_SHARE</code> if the scheduler does * not know about this application attempt. */ public int getAppFairShare(ApplicationAttemptId appAttemptId) { FSAppAttempt fsAppAttempt = scheduler.getSchedulerApp(appAttemptId); return fsAppAttempt == null ? INVALID_FAIR_SHARE : fsAppAttempt.getFairShare().getMemory(); }
private void tryPreemptMoreThanFairShare(String queueName) throws InterruptedException { ApplicationAttemptId appAttemptId = createSchedulingRequest(3 * GB, 3, queueName, "default", NODE_CAPACITY_MULTIPLE * rmNodes.size() / 2); starvingApp = scheduler.getSchedulerApp(appAttemptId); verifyPreemption(1, 5); }
/** * Submit an application to a given queue and preempt half resources of the * cluster. * * @param queueName queue name * @throws InterruptedException * if any thread has interrupted the current thread. */ private void preemptHalfResources(String queueName) throws InterruptedException { ApplicationAttemptId appAttemptId = createSchedulingRequest(2 * GB, 2, queueName, "default", NODE_CAPACITY_MULTIPLE * rmNodes.size() / 2); starvingApp = scheduler.getSchedulerApp(appAttemptId); // Move clock enough to identify starvation clock.tickSec(1); scheduler.update(); }
private void verifyAppRunnable(ApplicationAttemptId attId, boolean runnable) { FSAppAttempt app = scheduler.getSchedulerApp(attId); FSLeafQueue queue = app.getQueue(); assertEquals(runnable, queue.isRunnableApp(app)); assertEquals(!runnable, queue.isNonRunnableApp(app)); }
private void verifyAppRunnable(ApplicationAttemptId attId, boolean runnable) { FSAppAttempt app = scheduler.getSchedulerApp(attId); FSLeafQueue queue = app.getQueue(); assertEquals(runnable, queue.isRunnableApp(app)); assertEquals(!runnable, queue.isNonRunnableApp(app)); }
protected void warnOrKillContainer(RMContainer container) { ApplicationAttemptId appAttemptId = container.getApplicationAttemptId(); FSAppAttempt app = getSchedulerApp(appAttemptId); FSLeafQueue queue = app.getQueue(); LOG.info("Preempting container (prio=" + container.getContainer().getPriority() + "res=" + container.getContainer().getResource() + ") from queue " + queue.getName()); Long time = app.getContainerPreemptionTime(container); if (time != null) { // if we asked for preemption more than maxWaitTimeBeforeKill ms ago, // proceed with kill if (time + waitTimeBeforeKill < getClock().getTime()) { ContainerStatus status = SchedulerUtils.createPreemptedContainerStatus( container.getContainerId(), SchedulerUtils.PREEMPTED_CONTAINER); // TODO: Not sure if this ever actually adds this to the list of cleanup // containers on the RMNode (see SchedulerNode.releaseContainer()). completedContainer(container, status, RMContainerEventType.KILL); LOG.info("Killing container" + container + " (after waiting for premption for " + (getClock().getTime() - time) + "ms)"); } } else { // track the request in the FSAppAttempt itself app.addPreemption(container, getClock().getTime()); } }
protected void warnOrKillContainer(RMContainer container) { ApplicationAttemptId appAttemptId = container.getApplicationAttemptId(); FSAppAttempt app = getSchedulerApp(appAttemptId); FSLeafQueue queue = app.getQueue(); LOG.info("Preempting container (prio=" + container.getContainer().getPriority() + "res=" + container.getContainer().getResource() + ") from queue " + queue.getName()); Long time = app.getContainerPreemptionTime(container); if (time != null) { // if we asked for preemption more than maxWaitTimeBeforeKill ms ago, // proceed with kill if (time + waitTimeBeforeKill < getClock().getTime()) { ContainerStatus status = SchedulerUtils.createPreemptedContainerStatus( container.getContainerId(), SchedulerUtils.PREEMPTED_CONTAINER); // TODO: Not sure if this ever actually adds this to the list of cleanup // containers on the RMNode (see SchedulerNode.releaseContainer()). completedContainer(container, status, RMContainerEventType.KILL); LOG.info("Killing container" + container + " (after waiting for premption for " + (getClock().getTime() - time) + "ms)"); } } else { // track the request in the FSAppAttempt itself app.addPreemption(container, getClock().getTime()); } }
/** * Submit an application to a given queue and take over the entire cluster. * * @param queueName queue name */ private void takeAllResources(String queueName) { // Create an app that takes up all the resources on the cluster ApplicationAttemptId appAttemptId = createSchedulingRequest(GB, 1, queueName, "default", NODE_CAPACITY_MULTIPLE * rmNodes.size()); greedyApp = scheduler.getSchedulerApp(appAttemptId); scheduler.update(); sendEnoughNodeUpdatesToAssignFully(); assertEquals(8, greedyApp.getLiveContainers().size()); // Verify preemptable for queue and app attempt assertTrue( scheduler.getQueueManager().getQueue(queueName).isPreemptable() == greedyApp.isPreemptable()); }
@Test public void testEmptyQueueName() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // only default queue assertEquals(1, scheduler.getQueueManager().getLeafQueues().size()); // submit app with empty queue ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "", "user1"); scheduler.handle(appAddedEvent); // submission rejected assertEquals(1, scheduler.getQueueManager().getLeafQueues().size()); assertNull(scheduler.getSchedulerApp(appAttemptId)); assertEquals(0, resourceManager.getRMContext().getRMApps().size()); }
@Test public void testEmptyQueueName() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // only default queue assertEquals(1, scheduler.getQueueManager().getLeafQueues().size()); // submit app with empty queue ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "", "user1"); scheduler.handle(appAddedEvent); // submission rejected assertEquals(1, scheduler.getQueueManager().getLeafQueues().size()); assertNull(scheduler.getSchedulerApp(appAttemptId)); assertEquals(0, resourceManager.getRMContext().getRMApps().size()); }
@Test(timeout = 3000) public void testMaxAssignWithZeroMemoryContainers() throws Exception { conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(16384, 16), 0, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); scheduler.handle(nodeEvent); ApplicationAttemptId attId = createSchedulingRequest(0, 1, "root.default", "user", 8); FSAppAttempt app = scheduler.getSchedulerApp(attId); // set maxAssign to 2: only 2 containers should be allocated scheduler.maxAssign = 2; scheduler.update(); scheduler.handle(updateEvent); assertEquals("Incorrect number of containers allocated", 2, app .getLiveContainers().size()); // set maxAssign to -1: all remaining containers should be allocated scheduler.maxAssign = -1; scheduler.update(); scheduler.handle(updateEvent); assertEquals("Incorrect number of containers allocated", 8, app .getLiveContainers().size()); }
@Test (timeout = 60000) public void testBasic() throws InterruptedException { // Add one node String host = "127.0.0.1"; RMNode node1 = MockNodes.newNodeInfo( 1, Resources.createResource(4096, 4), 1, host); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); NodeUpdateSchedulerEvent nodeUpdateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(nodeUpdateEvent); ApplicationAttemptId appAttemptId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); createMockRMApp(appAttemptId); scheduler.addApplication(appAttemptId.getApplicationId(), "queue11", "user11", false); scheduler.addApplicationAttempt(appAttemptId, false, false); List<ResourceRequest> ask = new ArrayList<>(); ask.add(createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true)); scheduler.allocate( appAttemptId, ask, null, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS); FSAppAttempt app = scheduler.getSchedulerApp(appAttemptId); triggerSchedulingAttempt(); checkAppConsumption(app, Resources.createResource(1024, 1)); }
@Test public void testNoMoreCpuOnNode() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); ApplicationAttemptId attId = createSchedulingRequest(1024, 1, "default", "user1", 2); FSAppAttempt app = scheduler.getSchedulerApp(attId); scheduler.update(); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); }
@Test public void testNoMoreCpuOnNode() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); ApplicationAttemptId attId = createSchedulingRequest(1024, 1, "default", "user1", 2); FSAppAttempt app = scheduler.getSchedulerApp(attId); scheduler.update(); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); }
@Test (timeout = 5000) public void testMultipleContainersWaitingForReservation() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add a node RMNode node1 = MockNodes .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); // Request full capacity of node createSchedulingRequest(1024, "queue1", "user1", 1); scheduler.update(); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue2", "user2", 1); ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue3", "user3", 1); scheduler.update(); scheduler.handle(updateEvent); // One container should get reservation and the other should get nothing assertEquals(1024, scheduler.getSchedulerApp(attId1).getCurrentReservation().getMemory()); assertEquals(0, scheduler.getSchedulerApp(attId2).getCurrentReservation().getMemory()); }
@Test(timeout = 5000) public void testMultipleContainersWaitingForReservation() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add a node RMNode node1 = MockNodes .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); // Request full capacity of node createSchedulingRequest(1024, "queue1", "user1", 1); scheduler.update(); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue2", "user2", 1); ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue3", "user3", 1); scheduler.update(); scheduler.handle(updateEvent); // One container should get reservation and the other should get nothing assertEquals(1024, scheduler.getSchedulerApp(attId1).getCurrentReservation().getMemorySize()); assertEquals(0, scheduler.getSchedulerApp(attId2).getCurrentReservation().getMemorySize()); }
@Test public void testMoveMakesAppRunnable() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); QueueManager queueMgr = scheduler.getQueueManager(); FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0); ApplicationAttemptId appAttId = createSchedulingRequest(1024, 1, "queue1", "user1", 3); FSAppAttempt app = scheduler.getSchedulerApp(appAttId); assertTrue(oldQueue.isNonRunnableApp(app)); scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); assertFalse(oldQueue.isNonRunnableApp(app)); assertFalse(targetQueue.isNonRunnableApp(app)); assertTrue(targetQueue.isRunnableApp(app)); assertEquals(1, targetQueue.getNumRunnableApps()); assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps()); }
@Test public void testMoveMakesAppRunnable() throws Exception { QueueManager queueMgr = scheduler.getQueueManager(); FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); oldQueue.setMaxRunningApps(0); ApplicationAttemptId appAttId = createSchedulingRequest(1024, 1, "queue1", "user1", 3); FSAppAttempt app = scheduler.getSchedulerApp(appAttId); assertTrue(oldQueue.isNonRunnableApp(app)); scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); assertFalse(oldQueue.isNonRunnableApp(app)); assertFalse(targetQueue.isNonRunnableApp(app)); assertTrue(targetQueue.isRunnableApp(app)); assertEquals(1, targetQueue.getNumRunnableApps()); assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps()); } }