public FSQueue(String name, FairScheduler scheduler, FSParentQueue parent) { this.name = name; this.scheduler = scheduler; this.metrics = FSQueueMetrics.forQueue(getName(), parent, true, scheduler.getConf()); metrics.setMinShare(getMinShare()); metrics.setMaxShare(getMaxShare()); this.parent = parent; }
/** * Dump scheduler state including states of all queues. */ private void dumpSchedulerState() { FSQueue rootQueue = queueMgr.getRootQueue(); Resource clusterResource = getClusterResource(); STATE_DUMP_LOG.debug( "FairScheduler state: Cluster Capacity: " + clusterResource + " Allocations: " + rootMetrics.getAllocatedResources() + " Availability: " + Resource.newInstance( rootMetrics.getAvailableMB(), rootMetrics.getAvailableVirtualCores()) + " Demand: " + rootQueue.getDemand()); STATE_DUMP_LOG.debug(rootQueue.dumpState()); }
public synchronized static FSQueueMetrics forQueue(String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { MetricsSystem ms = DefaultMetricsSystem.instance(); QueueMetrics metrics = queueMetrics.get(queueName); if (metrics == null) { metrics = new FSQueueMetrics(ms, queueName, parent, enableUserMetrics, conf) .tag(QUEUE_INFO, queueName); // Register with the MetricsSystems if (ms != null) { metrics = ms.register( sourceName(queueName).toString(), "Metrics for queue: " + queueName, metrics); } queueMetrics.put(queueName, metrics); } return (FSQueueMetrics)metrics; }
/** * Subqueue metrics might be a little out of date because fair shares are * recalculated at the update interval, but the root queue metrics needs to * be updated synchronously with allocations and completions so that cluster * metrics will be consistent. */ private void updateRootQueueMetrics() { rootMetrics.setAvailableResourcesToQueue( Resources.subtract( clusterResource, rootMetrics.getAllocatedResources())); }
public Resource getReservedResource() { reservedResource.setMemorySize(metrics.getReservedMB()); reservedResource.setVirtualCores(metrics.getReservedVirtualCores()); return reservedResource; }
public QueueStatistics getQueueStatistics() { QueueStatistics stats = recordFactory.newRecordInstance(QueueStatistics.class); stats.setNumAppsSubmitted(getMetrics().getAppsSubmitted()); stats.setNumAppsRunning(getMetrics().getAppsRunning()); stats.setNumAppsPending(getMetrics().getAppsPending()); stats.setNumAppsCompleted(getMetrics().getAppsCompleted()); stats.setNumAppsKilled(getMetrics().getAppsKilled()); stats.setNumAppsFailed(getMetrics().getAppsFailed()); stats.setNumActiveUsers(getMetrics().getActiveUsers()); stats.setAvailableMemoryMB(getMetrics().getAvailableMB()); stats.setAllocatedMemoryMB(getMetrics().getAllocatedMB()); stats.setPendingMemoryMB(getMetrics().getPendingMB()); stats.setReservedMemoryMB(getMetrics().getReservedMB()); stats.setAvailableVCores(getMetrics().getAvailableVirtualCores()); stats.setAllocatedVCores(getMetrics().getAllocatedVirtualCores()); stats.setPendingVCores(getMetrics().getPendingVirtualCores()); stats.setReservedVCores(getMetrics().getReservedVirtualCores()); stats.setAllocatedContainers(getMetrics().getAllocatedContainers()); stats.setPendingContainers(getMetrics().getPendingContainers()); stats.setReservedContainers(getMetrics().getReservedContainers()); return stats; }
public synchronized static FSQueueMetrics forQueue(String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { MetricsSystem ms = DefaultMetricsSystem.instance(); return forQueue(ms, queueName, parent, enableUserMetrics, conf); }
/** * Check if preemption is enabled and the utilization threshold for * preemption is met. * * @return true if preemption should be attempted, false otherwise. */ private boolean shouldAttemptPreemption() { if (preemptionEnabled) { return (preemptionUtilizationThreshold < Math.max( (float) rootMetrics.getAllocatedMB() / clusterResource.getMemory(), (float) rootMetrics.getAllocatedVirtualCores() / clusterResource.getVirtualCores())); } return false; }
private void registerNodeAndSubmitApp( int memory, int vcores, int appContainers, int appMemory) { RMNode node1 = MockNodes.newNodeInfo( 1, Resources.createResource(memory, vcores), 1, "node1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); assertEquals("Incorrect amount of resources in the cluster", memory, scheduler.rootMetrics.getAvailableMB()); assertEquals("Incorrect amount of resources in the cluster", vcores, scheduler.rootMetrics.getAvailableVirtualCores()); createSchedulingRequest(appMemory, "queueA", "user1", appContainers); scheduler.update(); // Sufficient node check-ins to fully schedule containers for (int i = 0; i < 3; i++) { NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1); scheduler.handle(nodeUpdate1); } assertEquals("app1's request is not met", memory - appContainers * appMemory, scheduler.rootMetrics.getAvailableMB()); }
/** * Test if the metric scheduling policy is set correctly. */ @Test public void testSchedulingPolicy() { String queueName = "single"; FSQueueMetrics metrics = FSQueueMetrics.forQueue(ms, queueName, null, false, CONF); metrics.setSchedulingPolicy("drf"); checkSchedulingPolicy(queueName, "drf"); // test resetting the scheduling policy metrics.setSchedulingPolicy("fair"); checkSchedulingPolicy(queueName, "fair"); }
@Override public void setFairShare(Resource fairShare) { this.fairShare = fairShare; metrics.setFairShare(fairShare); }
assertEquals(1024, scheduler.rootMetrics.getReservedMB()); assertEquals(1024, scheduler.rootMetrics.getReservedMB()); assertEquals(0, scheduler.rootMetrics.getReservedMB());
@Before public void setUp() throws Exception { conf = new FairSchedulerConfiguration(); FairScheduler scheduler = mock(FairScheduler.class); AllocationConfiguration allocConf = new AllocationConfiguration(conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); SystemClock clock = new SystemClock(); when(scheduler.getClock()).thenReturn(clock); notEmptyQueues = new HashSet<FSQueue>(); queueManager = new QueueManager(scheduler) { @Override public boolean isEmpty(FSQueue queue) { return !notEmptyQueues.contains(queue); } }; FSQueueMetrics.forQueue("root", null, true, conf); queueManager.initialize(conf); }
/** * Subqueue metrics might be a little out of date because fair shares are * recalculated at the update interval, but the root queue metrics needs to * be updated synchronously with allocations and completions so that cluster * metrics will be consistent. */ private void updateRootQueueMetrics() { rootMetrics.setAvailableResourcesToQueue( Resources.subtract( clusterResource, rootMetrics.getAllocatedResources())); }
/** * Check if preemption is enabled and the utilization threshold for * preemption is met. * * @return true if preemption should be attempted, false otherwise. */ private boolean shouldAttemptPreemption() { if (preemptionEnabled) { return (preemptionUtilizationThreshold < Math.max( (float) rootMetrics.getAllocatedMB() / clusterResource.getMemory(), (float) rootMetrics.getAllocatedVirtualCores() / clusterResource.getVirtualCores())); } return false; }
@Override public void setFairShare(Resource fairShare) { this.fairShare = fairShare; metrics.setFairShare(fairShare); }
assertEquals(1024, scheduler.rootMetrics.getReservedMB()); assertEquals(1024, scheduler.rootMetrics.getReservedMB()); assertEquals(0, scheduler.rootMetrics.getReservedMB());
/** * Recompute the internal variables used by the scheduler - per-job weights, * fair shares, deficits, minimum slot allocations, and amount of used and * required resources per job. */ protected synchronized void update() { long start = getClock().getTime(); updateStarvationStats(); // Determine if any queues merit preemption FSQueue rootQueue = queueMgr.getRootQueue(); // Recursively update demands for all queues rootQueue.updateDemand(); rootQueue.setFairShare(clusterResource); // Recursively compute fair shares for all queues // and update metrics rootQueue.recomputeShares(); updateRootQueueMetrics(); if (LOG.isDebugEnabled()) { if (--updatesToSkipForDebug < 0) { updatesToSkipForDebug = UPDATE_DEBUG_FREQUENCY; LOG.debug("Cluster Capacity: " + clusterResource + " Allocations: " + rootMetrics.getAllocatedResources() + " Availability: " + Resource.newInstance( rootMetrics.getAvailableMB(), rootMetrics.getAvailableVirtualCores()) + " Demand: " + rootQueue.getDemand()); } } long duration = getClock().getTime() - start; fsOpDurations.addUpdateCallDuration(duration); }
public synchronized static FSQueueMetrics forQueue(String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { MetricsSystem ms = DefaultMetricsSystem.instance(); QueueMetrics metrics = queueMetrics.get(queueName); if (metrics == null) { metrics = new FSQueueMetrics(ms, queueName, parent, enableUserMetrics, conf) .tag(QUEUE_INFO, queueName); // Register with the MetricsSystems if (ms != null) { metrics = ms.register( sourceName(queueName).toString(), "Metrics for queue: " + queueName, metrics); } queueMetrics.put(queueName, metrics); } return (FSQueueMetrics)metrics; }
public FSQueue(String name, FairScheduler scheduler, FSParentQueue parent) { this.name = name; this.scheduler = scheduler; this.metrics = FSQueueMetrics.forQueue(getName(), parent, true, scheduler.getConf()); metrics.setMinShare(getMinShare()); metrics.setMaxShare(getMaxShare()); this.parent = parent; }