/** * Dump scheduler state including states of all queues. */ private void dumpSchedulerState() { FSQueue rootQueue = queueMgr.getRootQueue(); Resource clusterResource = getClusterResource(); STATE_DUMP_LOG.debug( "FairScheduler state: Cluster Capacity: " + clusterResource + " Allocations: " + rootMetrics.getAllocatedResources() + " Availability: " + Resource.newInstance( rootMetrics.getAvailableMB(), rootMetrics.getAvailableVirtualCores()) + " Demand: " + rootQueue.getDemand()); STATE_DUMP_LOG.debug(rootQueue.dumpState()); }
/** * Recompute the internal variables used by the scheduler - per-job weights, * fair shares, deficits, minimum slot allocations, and amount of used and * required resources per job. */ protected synchronized void update() { long start = getClock().getTime(); updateStarvationStats(); // Determine if any queues merit preemption FSQueue rootQueue = queueMgr.getRootQueue(); // Recursively update demands for all queues rootQueue.updateDemand(); rootQueue.setFairShare(clusterResource); // Recursively compute fair shares for all queues // and update metrics rootQueue.recomputeShares(); updateRootQueueMetrics(); if (LOG.isDebugEnabled()) { if (--updatesToSkipForDebug < 0) { updatesToSkipForDebug = UPDATE_DEBUG_FREQUENCY; LOG.debug("Cluster Capacity: " + clusterResource + " Allocations: " + rootMetrics.getAllocatedResources() + " Availability: " + Resource.newInstance( rootMetrics.getAvailableMB(), rootMetrics.getAvailableVirtualCores()) + " Demand: " + rootQueue.getDemand()); } } long duration = getClock().getTime() - start; fsOpDurations.addUpdateCallDuration(duration); }
/** * Recompute the internal variables used by the scheduler - per-job weights, * fair shares, deficits, minimum slot allocations, and amount of used and * required resources per job. */ protected synchronized void update() { long start = getClock().getTime(); updateStarvationStats(); // Determine if any queues merit preemption FSQueue rootQueue = queueMgr.getRootQueue(); // Recursively update demands for all queues rootQueue.updateDemand(); rootQueue.setFairShare(clusterResource); // Recursively compute fair shares for all queues // and update metrics rootQueue.recomputeShares(); updateRootQueueMetrics(); if (LOG.isDebugEnabled()) { if (--updatesToSkipForDebug < 0) { updatesToSkipForDebug = UPDATE_DEBUG_FREQUENCY; LOG.debug("Cluster Capacity: " + clusterResource + " Allocations: " + rootMetrics.getAllocatedResources() + " Availability: " + Resource.newInstance( rootMetrics.getAvailableMB(), rootMetrics.getAvailableVirtualCores()) + " Demand: " + rootQueue.getDemand()); } } long duration = getClock().getTime() - start; fsOpDurations.addUpdateCallDuration(duration); }
private void registerNodeAndSubmitApp( int memory, int vcores, int appContainers, int appMemory) { RMNode node1 = MockNodes.newNodeInfo( 1, Resources.createResource(memory, vcores), 1, "node1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); assertEquals("Incorrect amount of resources in the cluster", memory, scheduler.rootMetrics.getAvailableMB()); assertEquals("Incorrect amount of resources in the cluster", vcores, scheduler.rootMetrics.getAvailableVirtualCores()); createSchedulingRequest(appMemory, "queueA", "user1", appContainers); scheduler.update(); // Sufficient node check-ins to fully schedule containers for (int i = 0; i < 3; i++) { NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1); scheduler.handle(nodeUpdate1); } assertEquals("app1's request is not met", memory - appContainers * appMemory, scheduler.rootMetrics.getAvailableMB()); }
public QueueStatistics getQueueStatistics() { QueueStatistics stats = recordFactory.newRecordInstance(QueueStatistics.class); stats.setNumAppsSubmitted(getMetrics().getAppsSubmitted()); stats.setNumAppsRunning(getMetrics().getAppsRunning()); stats.setNumAppsPending(getMetrics().getAppsPending()); stats.setNumAppsCompleted(getMetrics().getAppsCompleted()); stats.setNumAppsKilled(getMetrics().getAppsKilled()); stats.setNumAppsFailed(getMetrics().getAppsFailed()); stats.setNumActiveUsers(getMetrics().getActiveUsers()); stats.setAvailableMemoryMB(getMetrics().getAvailableMB()); stats.setAllocatedMemoryMB(getMetrics().getAllocatedMB()); stats.setPendingMemoryMB(getMetrics().getPendingMB()); stats.setReservedMemoryMB(getMetrics().getReservedMB()); stats.setAvailableVCores(getMetrics().getAvailableVirtualCores()); stats.setAllocatedVCores(getMetrics().getAllocatedVirtualCores()); stats.setPendingVCores(getMetrics().getPendingVirtualCores()); stats.setReservedVCores(getMetrics().getReservedVirtualCores()); stats.setAllocatedContainers(getMetrics().getAllocatedContainers()); stats.setPendingContainers(getMetrics().getPendingContainers()); stats.setReservedContainers(getMetrics().getReservedContainers()); return stats; }