@Test (expected = YarnException.class) public void testMoveWouldViolateMaxAppsConstraints() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); QueueManager queueMgr = scheduler.getQueueManager(); queueMgr.getLeafQueue("queue2", true); scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0); ApplicationAttemptId appAttId = createSchedulingRequest(1024, 1, "queue1", "user1", 3); scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); }
/** * Get the fair share assigned to the appAttemptId. * @param appAttemptId * @return The fair share assigned to the appAttemptId, * <code>FairSchedulerInfo#INVALID_FAIR_SHARE</code> if the scheduler does * not know about this application attempt. */ public long getAppFairShare(ApplicationAttemptId appAttemptId) { FSAppAttempt fsAppAttempt = scheduler.getSchedulerApp(appAttemptId); return fsAppAttempt == null ? INVALID_FAIR_SHARE : fsAppAttempt.getFairShare().getMemorySize(); }
/** * Helper method for the tests to assign the app to a queue. */ @VisibleForTesting FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user) { return assignToQueue(rmApp, queueName, user, null); }
@Override public void run() { while (!Thread.currentThread().isInterrupted()) { try { continuousSchedulingAttempt(); Thread.sleep(getContinuousSchedulingSleepMs()); } catch (InterruptedException e) { LOG.warn("Continuous scheduling thread interrupted. Exiting.", e); return; } } } }
@Override protected void nodeUpdate(RMNode nm) { writeLock.lock(); try { long start = getClock().getTime(); super.nodeUpdate(nm); FSSchedulerNode fsNode = getFSSchedulerNode(nm.getNodeID()); attemptScheduling(fsNode); long duration = getClock().getTime() - start; fsOpDurations.addNodeUpdateDuration(duration); } finally { writeLock.unlock(); } }
public FSQueue(String name, FairScheduler scheduler, FSParentQueue parent) { this.name = name; this.scheduler = scheduler; this.authorizer = YarnAuthorizationProvider.getInstance(scheduler.getConf()); this.queueEntity = new PrivilegedEntity(EntityType.QUEUE, name); this.metrics = FSQueueMetrics.forQueue(getName(), parent, true, scheduler.getConf()); this.parent = parent; setPolicy(scheduler.getAllocationConfiguration().getSchedulingPolicy(name)); reinit(false); }
@Test public void testCompletedContainerOnRemovedNode() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); scheduler.handle(new NodeAddedSchedulerEvent(node)); scheduler.addApplication(appAttemptId.getApplicationId(), "root.queue1", "user1", false); scheduler.addApplicationAttempt(appAttemptId, false, false); scheduler.allocate(appAttemptId, ask1, null, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS); scheduler.update(); scheduler.handle(new NodeUpdateSchedulerEvent(node)); Collection<RMContainer> clist = scheduler.getSchedulerApp(appAttemptId) .getLiveContainers(); Assert.assertEquals(1, clist.size()); scheduler.handle(new NodeRemovedSchedulerEvent(node)); scheduler.completedContainer(rmc, SchedulerUtils.createAbnormalContainerStatus(rmc.getContainerId(), SchedulerUtils.COMPLETED_APPLICATION),
@Test public void testAssignToNonLeafQueueReturnsNull() throws Exception { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); scheduler.getQueueManager().getLeafQueue("root.child1.granchild", true); scheduler.getQueueManager().getLeafQueue("root.child2", true); RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW); // Trying to assign to non leaf queue would return null assertNull(scheduler.assignToQueue(rmApp1, "root.child1", "tintin")); assertNotNull(scheduler.assignToQueue(rmApp2, "root.child2", "snowy")); }
@Test (timeout = 60000) public void testBasic() throws InterruptedException { // Add one node String host = "127.0.0.1"; RMNode node1 = MockNodes.newNodeInfo( 1, Resources.createResource(4096, 4), 1, host); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); NodeUpdateSchedulerEvent nodeUpdateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(nodeUpdateEvent); ApplicationAttemptId appAttemptId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); createMockRMApp(appAttemptId); scheduler.addApplication(appAttemptId.getApplicationId(), "queue11", "user11", false); scheduler.addApplicationAttempt(appAttemptId, false, false); List<ResourceRequest> ask = new ArrayList<>(); ask.add(createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true)); scheduler.allocate( appAttemptId, ask, null, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS); FSAppAttempt app = scheduler.getSchedulerApp(appAttemptId); triggerSchedulingAttempt(); checkAppConsumption(app, Resources.createResource(1024, 1)); }
public void testSchedulingOnRemovedNode() throws Exception { scheduler.init(conf); scheduler.start(); Assert.assertTrue("Continuous scheduling should be disabled.", !scheduler.isContinuousSchedulingEnabled()); scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1", false); scheduler.addApplicationAttempt(id11, false, false); scheduler.allocate(id11, ask1, null, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS); Resources.createResource(8 * 1024, 8), 1, hostName); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); FSSchedulerNode node = (FSSchedulerNode)scheduler.getSchedulerNode( node1.getNodeID()); scheduler.handle(removeNode1); scheduler.attemptScheduling(node); scheduler.handle(appRemovedEvent1);
scheduler = new FairScheduler(); conf = super.createConfiguration(); resourceManager = new MockRM(conf); .rollMasterKey(); scheduler.setRMContext(resourceManager.getRMContext()); scheduler.init(conf); scheduler.start(); FairScheduler spyScheduler = spy(scheduler); Assert.assertTrue("Continuous scheduling should be disabled.", !spyScheduler.isContinuousSchedulingEnabled()); "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); spyScheduler.handle(nodeEvent1); Assert.assertEquals("We should have one alive node.", 1, spyScheduler.getNumClusterNodes()); InterruptedException ie = new InterruptedException(); doThrow(new YarnRuntimeException(ie)).when(spyScheduler). attemptScheduling(isA(FSSchedulerNode.class)); spyScheduler.continuousSchedulingAttempt(); fail("Expected InterruptedException to stop schedulingThread"); } catch (InterruptedException e) {
@Test public void testLowestCommonAncestorForNonRootParent() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); FSLeafQueue aQueue = mock(FSLeafQueue.class); FSLeafQueue bQueue = mock(FSLeafQueue.class); when(aQueue.getName()).thenReturn("root.queue1.a"); when(bQueue.getName()).thenReturn("root.queue1.b"); QueueManager queueManager = scheduler.getQueueManager(); FSParentQueue queue1 = queueManager.getParentQueue("queue1", true); queue1.addChildQueue(aQueue); queue1.addChildQueue(bQueue); FSQueue ancestorQueue = scheduler.findLowestCommonAncestorQueue(aQueue, bQueue); assertEquals(ancestorQueue, queue1); }
@Test public void testContinuousSchedulingInterruptedException() throws Exception { scheduler.init(conf); scheduler.start(); FairScheduler spyScheduler = spy(scheduler); Assert.assertTrue("Continuous scheduling should be disabled.", !spyScheduler.isContinuousSchedulingEnabled()); // Add one nodes RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); spyScheduler.handle(nodeEvent1); Assert.assertEquals("We should have one alive node.", 1, spyScheduler.getNumClusterNodes()); InterruptedException ie = new InterruptedException(); doThrow(new YarnRuntimeException(ie)).when(spyScheduler). attemptScheduling(isA(FSSchedulerNode.class)); // Invoke the continuous scheduling once try { spyScheduler.continuousSchedulingAttempt(); fail("Expected InterruptedException to stop schedulingThread"); } catch (InterruptedException e) { Assert.assertEquals(ie, e); } }
protected ApplicationAttemptId createSchedulingRequest(String queueId, String userId, List<ResourceRequest> ask) { ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); scheduler.addApplication(id.getApplicationId(), queueId, userId, false); // This conditional is for testAclSubmitApplication where app is rejected // and no app is added. if (scheduler.getSchedulerApplications().containsKey(id.getApplicationId())) { scheduler.addApplicationAttempt(id, false, false); } RMApp rmApp = mock(RMApp.class); RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class); when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt); when(rmAppAttempt.getRMAppAttemptMetrics()).thenReturn( new RMAppAttemptMetrics(id,resourceManager.getRMContext())); resourceManager.getRMContext().getRMApps() .put(id.getApplicationId(), rmApp); scheduler.allocate(id, ask, new ArrayList<ContainerId>(), null, null); return id; }
scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); int minReqSize = FairSchedulerConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB; scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1", false); scheduler.addApplicationAttempt(id11, false, false); List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>(); ResourceRequest request1 = createResourceRequest(minReqSize * 2, ResourceRequest.ANY, 1, 1, true); ask1.add(request1); scheduler.allocate(id11, ask1, new ArrayList<ContainerId>(), null, null); scheduler.addApplication(id21.getApplicationId(), "root.queue2", "user1", false); scheduler.addApplicationAttempt(id21, false, false); List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>(); ResourceRequest request2 = createResourceRequest(2 * minReqSize, ask2.add(request2); ask2.add(request3); scheduler.allocate(id21, ask2, new ArrayList<ContainerId>(), null, null); scheduler.addApplication(id22.getApplicationId(), "root.queue2", "user1", false); scheduler.addApplicationAttempt(id22, false, false); List<ResourceRequest> ask3 = new ArrayList<ResourceRequest>();
long start = getClock().getTime(); if (LOG.isDebugEnabled()) { LOG.debug("nodeUpdate: " + nm + " cluster capacity: " + clusterResource); FSSchedulerNode node = getFSSchedulerNode(nm.getNodeID()); containerLaunchedOnNode(launchedContainer.getContainerId(), node); ContainerId containerId = completedContainer.getContainerId(); LOG.debug("Container FINISHED: " + containerId); completedContainer(getRMContainer(containerId), completedContainer, RMContainerEventType.FINISHED); attemptScheduling(node); attemptScheduling(node); long duration = getClock().getTime() - start; fsOpDurations.addNodeUpdateDuration(duration);
@Test public void testFairSchedulerContinuousSchedulingInitTime() throws Exception { scheduler.start(); createMockRMApp(id11); priority = Priority.newInstance(priorityValue); scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1", false); scheduler.addApplicationAttempt(id11, false, false); fsAppAttempt = scheduler.getApplicationAttempt(id11); ask1.add(request1); ask1.add(request2); scheduler.allocate(id11, ask1, null, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS); scheduler.handle(nodeEvent1); FSSchedulerNode node = (FSSchedulerNode) scheduler.getSchedulerNode(node1.getNodeID()); scheduler.attemptScheduling(node); Map<SchedulerRequestKey, Long> lastScheduledContainer = fsAppAttempt.getLastScheduledContainer();
@Test public void testEmptyQueueName() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // only default queue assertEquals(1, scheduler.getQueueManager().getLeafQueues().size()); // submit app with empty queue ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "", "user1"); scheduler.handle(appAddedEvent); // submission rejected assertEquals(1, scheduler.getQueueManager().getLeafQueues().size()); assertNull(scheduler.getSchedulerApp(appAttemptId)); assertEquals(0, resourceManager.getRMContext().getRMApps().size()); }
@Test public void testNoMoreCpuOnNode() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); ApplicationAttemptId attId = createSchedulingRequest(1024, 1, "default", "user1", 2); FSAppAttempt app = scheduler.getSchedulerApp(attId); scheduler.update(); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); }
@Before public void setup() { mockClock = new MockClock(); conf = createConfiguration(); resourceManager = new MockRM(conf); resourceManager.start(); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); scheduler.setClock(mockClock); assertTrue(scheduler.isContinuousSchedulingEnabled()); assertEquals( FairSchedulerConfiguration.DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS, scheduler.getContinuousSchedulingSleepMs()); assertEquals(mockClock, scheduler.getClock()); }