private void doneApplication(ApplicationId applicationId, RMAppState finalState) { try { writeLock.lock(); SchedulerApplication<FiCaSchedulerApp> application = applications.get( applicationId); if (application == null) { // The AppRemovedSchedulerEvent maybe sent on recovery for completed // apps, ignore it. LOG.warn("Couldn't find application " + applicationId); return; } CSQueue queue = (CSQueue) application.getQueue(); if (!(queue instanceof LeafQueue)) { LOG.error("Cannot finish application " + "from non-leaf queue: " + queue .getQueueName()); } else{ queue.finishApplication(applicationId, application.getUser()); } application.stop(finalState); applications.remove(applicationId); } finally { writeLock.unlock(); } }
private synchronized void doneApplication(ApplicationId applicationId, RMAppState finalState) { SchedulerApplication<FiCaSchedulerApp> application = applications.get(applicationId); if (application == null){ // The AppRemovedSchedulerEvent maybe sent on recovery for completed apps, // ignore it. LOG.warn("Couldn't find application " + applicationId); return; } CSQueue queue = (CSQueue) application.getQueue(); if (!(queue instanceof LeafQueue)) { LOG.error("Cannot finish application " + "from non-leaf queue: " + queue.getQueueName()); } else { queue.finishApplication(applicationId, application.getUser()); } application.stop(finalState); applications.remove(applicationId); }
private synchronized void doneApplication(ApplicationId applicationId, RMAppState finalState) { SchedulerApplication<FiCaSchedulerApp> application = applications.get(applicationId); if (application == null){ // The AppRemovedSchedulerEvent maybe sent on recovery for completed apps, // ignore it. LOG.warn("Couldn't find application " + applicationId); return; } CSQueue queue = (CSQueue) application.getQueue(); if (!(queue instanceof LeafQueue)) { LOG.error("Cannot finish application " + "from non-leaf queue: " + queue.getQueueName()); } else { queue.finishApplication(applicationId, application.getUser()); } application.stop(finalState); applications.remove(applicationId); }
@Override public void preValidateMoveApplication(ApplicationId appId, String newQueue) throws YarnException { writeLock.lock(); try { SchedulerApplication<FSAppAttempt> app = applications.get(appId); if (app == null) { throw new YarnException("App to be moved " + appId + " not found."); } FSAppAttempt attempt = app.getCurrentAppAttempt(); // To serialize with FairScheduler#allocate, synchronize on app attempt attempt.getWriteLock().lock(); try { FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue(); String destQueueName = handleMoveToPlanQueue(newQueue); FSLeafQueue targetQueue = queueMgr.getLeafQueue(destQueueName, false); if (targetQueue == null) { throw new YarnException("Target queue " + newQueue + " not found or is not a leaf queue."); } if (oldQueue.isRunnableApp(attempt)) { verifyMoveDoesNotViolateConstraints(attempt, oldQueue, targetQueue); } } finally { attempt.getWriteLock().unlock(); } } finally { writeLock.unlock(); } }
throw new YarnException("App to be moved " + appId + " not found."); String sourceQueueName = application.getQueue().getQueueName(); this.queueManager.getAndCheckLeafQueue(sourceQueueName); String destQueueName = handleMoveToPlanQueue(newQueue);
return; CSQueue queue = (CSQueue) application.getQueue();
@SuppressWarnings("static-access") public void clearQueueMetrics(RMApp app) { ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) getResourceScheduler()) .getSchedulerApplications().get(app.getApplicationId()).getQueue() .getMetrics().clearQueueMetrics(); }
@SuppressWarnings("static-access") public void clearQueueMetrics(RMApp app) { ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) getResourceScheduler()) .getSchedulerApplications().get(app.getApplicationId()).getQueue() .getMetrics().clearQueueMetrics(); }
return; CSQueue queue = (CSQueue) application.getQueue();
@Override public synchronized String moveApplication(ApplicationId appId, String queueName) throws YarnException { SchedulerApplication<FSAppAttempt> app = applications.get(appId); if (app == null) { throw new YarnException("App to be moved " + appId + " not found."); } FSAppAttempt attempt = (FSAppAttempt) app.getCurrentAppAttempt(); // To serialize with FairScheduler#allocate, synchronize on app attempt synchronized (attempt) { FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue(); String destQueueName = handleMoveToPlanQueue(queueName); FSLeafQueue targetQueue = queueMgr.getLeafQueue(destQueueName, false); if (targetQueue == null) { throw new YarnException("Target queue " + queueName + " not found or is not a leaf queue."); } if (targetQueue == oldQueue) { return oldQueue.getQueueName(); } if (oldQueue.isRunnableApp(attempt)) { verifyMoveDoesNotViolateConstraints(attempt, oldQueue, targetQueue); } executeMove(app, attempt, oldQueue, targetQueue); return targetQueue.getQueueName(); } }
@Override public synchronized String moveApplication(ApplicationId appId, String queueName) throws YarnException { SchedulerApplication<FSAppAttempt> app = applications.get(appId); if (app == null) { throw new YarnException("App to be moved " + appId + " not found."); } FSAppAttempt attempt = (FSAppAttempt) app.getCurrentAppAttempt(); // To serialize with FairScheduler#allocate, synchronize on app attempt synchronized (attempt) { FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue(); String destQueueName = handleMoveToPlanQueue(queueName); FSLeafQueue targetQueue = queueMgr.getLeafQueue(destQueueName, false); if (targetQueue == null) { throw new YarnException("Target queue " + queueName + " not found or is not a leaf queue."); } if (targetQueue == oldQueue) { return oldQueue.getQueueName(); } if (oldQueue.isRunnableApp(attempt)) { verifyMoveDoesNotViolateConstraints(attempt, oldQueue, targetQueue); } executeMove(app, attempt, oldQueue, targetQueue); return targetQueue.getQueueName(); } }
applications.get(applicationAttemptId.getApplicationId()); String user = application.getUser(); FSLeafQueue queue = (FSLeafQueue) application.getQueue();
private void checkAppQueue(MockRM resourceManager, String user, String submissionQueue, String expected) throws Exception { RMApp app = resourceManager.submitApp(200, "name", user, new HashMap<ApplicationAccessType, String>(), false, submissionQueue, -1, null, "MAPREDUCE", false); RMAppState expectedState = expected.isEmpty() ? RMAppState.FAILED : RMAppState.ACCEPTED; resourceManager.waitForState(app.getApplicationId(), expectedState); // get scheduler app CapacityScheduler cs = (CapacityScheduler) resourceManager.getResourceScheduler(); SchedulerApplication schedulerApp = cs.getSchedulerApplications().get(app.getApplicationId()); String queue = ""; if (schedulerApp != null) { queue = schedulerApp.getQueue().getQueueName(); } Assert.assertTrue("expected " + expected + " actual " + queue, expected.equals(queue)); Assert.assertEquals(expected, app.getQueue()); }
@Test public void testAddAndRemoveAppFromCapacityScheduler() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); MockRM rm = new MockRM(conf); @SuppressWarnings("unchecked") AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> cs = (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm .getResourceScheduler(); SchedulerApplication<SchedulerApplicationAttempt> app = TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler( cs.getSchedulerApplications(), cs, "a1"); Assert.assertEquals("a1", app.getQueue().getQueueName()); }
@Test public void testAddAndRemoveAppFromCapacityScheduler() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); MockRM rm = new MockRM(conf); @SuppressWarnings("unchecked") AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> cs = (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm .getResourceScheduler(); SchedulerApplication<SchedulerApplicationAttempt> app = TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler( cs.getSchedulerApplications(), cs, "a1"); Assert.assertEquals("a1", app.getQueue().getQueueName()); }
private void checkCSLeafQueue(MockRM rm, SchedulerApplication<SchedulerApplicationAttempt> app, Resource clusterResource, Resource queueResource, Resource usedResource, int numContainers) { LeafQueue leafQueue = (LeafQueue) app.getQueue(); // assert queue used resources. assertEquals(usedResource, leafQueue.getUsedResources()); assertEquals(numContainers, leafQueue.getNumContainers()); ResourceCalculator calc = ((CapacityScheduler) rm.getResourceScheduler()).getResourceCalculator(); float usedCapacity = Resources.divide(calc, clusterResource, usedResource, queueResource); // assert queue used capacity assertEquals(usedCapacity, leafQueue.getUsedCapacity(), 1e-8); float absoluteUsedCapacity = Resources.divide(calc, clusterResource, usedResource, clusterResource); // assert queue absolute capacity assertEquals(absoluteUsedCapacity, leafQueue.getAbsoluteUsedCapacity(), 1e-8); // assert user consumed resources. assertEquals(usedResource, leafQueue.getUser(app.getUser()) .getUsed()); }
private void checkCSLeafQueue(MockRM rm, SchedulerApplication<SchedulerApplicationAttempt> app, Resource clusterResource, Resource queueResource, Resource usedResource, int numContainers) { LeafQueue leafQueue = (LeafQueue) app.getQueue(); // assert queue used resources. assertEquals(usedResource, leafQueue.getUsedResources()); assertEquals(numContainers, leafQueue.getNumContainers()); ResourceCalculator calc = ((CapacityScheduler) rm.getResourceScheduler()).getResourceCalculator(); float usedCapacity = Resources.divide(calc, clusterResource, usedResource, queueResource); // assert queue used capacity assertEquals(usedCapacity, leafQueue.getUsedCapacity(), 1e-8); float absoluteUsedCapacity = Resources.divide(calc, clusterResource, usedResource, clusterResource); // assert queue absolute capacity assertEquals(absoluteUsedCapacity, leafQueue.getAbsoluteUsedCapacity(), 1e-8); // assert user consumed resources. assertEquals(usedResource, leafQueue.getUser(app.getUser()) .getUsed()); }
private void checkCSQueue(MockRM rm, SchedulerApplication<SchedulerApplicationAttempt> app, Resource clusterResource, Resource queueResource, Resource usedResource, int numContainers) throws Exception { checkCSLeafQueue(rm, app, clusterResource, queueResource, usedResource, numContainers); LeafQueue queue = (LeafQueue) app.getQueue(); Resource availableResources = Resources.subtract(queueResource, usedResource); // ************ check app headroom **************** SchedulerApplicationAttempt schedulerAttempt = app.getCurrentAppAttempt(); assertEquals(availableResources, schedulerAttempt.getHeadroom()); // ************* check Queue metrics ************ QueueMetrics queueMetrics = queue.getMetrics(); assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemorySize(), availableResources.getVirtualCores(), usedResource.getMemorySize(), usedResource.getVirtualCores()); // ************ check user metrics *********** QueueMetrics userMetrics = queueMetrics.getUserMetrics(app.getUser()); assertMetrics(userMetrics, 1, 0, 1, 0, 2, availableResources.getMemorySize(), availableResources.getVirtualCores(), usedResource.getMemorySize(), usedResource.getVirtualCores()); }
private void checkCSQueue(MockRM rm, SchedulerApplication<SchedulerApplicationAttempt> app, Resource clusterResource, Resource queueResource, Resource usedResource, int numContainers) throws Exception { checkCSLeafQueue(rm, app, clusterResource, queueResource, usedResource, numContainers); LeafQueue queue = (LeafQueue) app.getQueue(); Resource availableResources = Resources.subtract(queueResource, usedResource); // ************ check app headroom **************** SchedulerApplicationAttempt schedulerAttempt = app.getCurrentAppAttempt(); assertEquals(availableResources, schedulerAttempt.getHeadroom()); // ************* check Queue metrics ************ QueueMetrics queueMetrics = queue.getMetrics(); assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(), availableResources.getVirtualCores(), usedResource.getMemory(), usedResource.getVirtualCores()); // ************ check user metrics *********** QueueMetrics userMetrics = queueMetrics.getUserMetrics(app.getUser()); assertMetrics(userMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(), availableResources.getVirtualCores(), usedResource.getMemory(), usedResource.getVirtualCores()); }
schedulerApp.getCurrentAppAttempt().getAMResource()); FSQueueMetrics fsQueueMetrics = (FSQueueMetrics) schedulerApp.getQueue().getMetrics(); assertEquals(amResources.getMemorySize(), fsQueueMetrics.getAMResourceUsageMB());