/** Convenient toString implementation for debugging. */ @Override public String toString() { return String.format("[%s, demand=%s, running=%s, share=%s, w=%s]", getName(), getDemand(), getResourceUsage(), fairShare, getWeights()); }
/** * Remove a queue and all its descendents. */ private void removeQueue(FSQueue queue) { if (queue instanceof FSLeafQueue) { leafQueues.remove(queue); } else { List<FSQueue> childQueues = queue.getChildQueues(); while (!childQueues.isEmpty()) { removeQueue(childQueues.get(0)); } } queues.remove(queue.getName()); queue.getParent().getChildQueues().remove(queue); }
/** * Initialize a {@link FSQueue} with queue-specific properties and its * metrics. * @param queue the FSQueue needed to be initialized */ public void initFSQueue(FSQueue queue){ // Set queue-specific properties. String name = queue.getName(); queue.setWeights(getQueueWeight(name)); queue.setMinShare(getMinResources(name)); queue.setMaxShare(getMaxResources(name)); queue.setMaxRunningApps(getQueueMaxApps(name)); queue.setMaxAMShare(getQueueMaxAMShare(name)); queue.setMaxChildQueueResource(getMaxChildResources(name)); queue.setMaxContainerAllocation(getQueueMaxContainerAllocation(name)); // Set queue metrics. queue.getMetrics().setMinShare(queue.getMinShare()); queue.getMetrics().setMaxShare(queue.getMaxShare()); queue.getMetrics().setMaxApps(queue.getMaxRunningApps()); queue.getMetrics().setSchedulingPolicy(getSchedulingPolicy(name).getName()); } }
/** Convenient toString implementation for debugging. */ @Override public String toString() { return String.format("[%s, demand=%s, running=%s, share=%s, w=%s]", getName(), getDemand(), getResourceUsage(), fairShare, getWeight()); }
boolean fitsInMaxShare(Resource additionalResource) { Resource usagePlusAddition = Resources.add(getResourceUsage(), additionalResource); if (!Resources.fitsIn(usagePlusAddition, getMaxShare())) { if (LOG.isDebugEnabled()) { LOG.debug("Resource usage plus resource request: " + usagePlusAddition + " exceeds maximum resource allowed:" + getMaxShare() + " in queue " + getName()); } return false; } FSQueue parentQueue = getParent(); if (parentQueue != null) { return parentQueue.fitsInMaxShare(additionalResource); } return true; }
public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) { AllocationConfiguration allocConf = scheduler.getAllocationConfiguration(); queueName = queue.getName(); schedulingPolicy = queue.getPolicy().getName(); usedResources = new ResourceInfo(queue.getResourceUsage()); fractionMemUsed = (float)usedResources.getMemory() / clusterResources.getMemory(); steadyFairResources = new ResourceInfo(queue.getSteadyFairShare()); fairResources = new ResourceInfo(queue.getFairShare()); minResources = new ResourceInfo(queue.getMinShare()); maxResources = new ResourceInfo(queue.getMaxShare()); maxResources = new ResourceInfo( Resources.componentwiseMin(queue.getMaxShare(), scheduler.getClusterResource())); Collection<FSQueue> children = queue.getChildQueues(); for (FSQueue child : children) { if (child instanceof FSLeafQueue) {
queueName = queue.getName(); schedulingPolicy = queue.getPolicy().getName(); queue.getMetrics().getAMResourceUsageMB(), queue.getMetrics().getAMResourceUsageVCores())); amMaxResources = new ResourceInfo(Resource.newInstance( queue.getMetrics().getMaxAMShareMB(), queue.getMetrics().getMaxAMShareVCores())); usedResources = new ResourceInfo(queue.getResourceUsage()); demandResources = new ResourceInfo(queue.getDemand()); fractionMemUsed = (float)usedResources.getMemorySize() / clusterResources.getMemorySize(); steadyFairResources = new ResourceInfo(queue.getSteadyFairShare()); fairResources = new ResourceInfo(queue.getFairShare()); minResources = new ResourceInfo(queue.getMinShare()); maxResources = new ResourceInfo( Resources.componentwiseMin(queue.getMaxShare(), scheduler.getClusterResource())); maxContainerAllocation = new ResourceInfo(scheduler.getMaximumResourceCapability(queueName)); reservedResources = new ResourceInfo(queue.getReservedResource()); maxApps = queue.getMaxRunningApps(); allocatedContainers = queue.getMetrics().getAllocatedContainers(); reservedContainers = queue.getMetrics().getReservedContainers(); preemptable = queue.isPreemptable();
@Override public QueueInfo getQueueInfo(boolean includeChildQueues, boolean recursive) { QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class); queueInfo.setQueueName(getQueueName()); if (scheduler.getClusterResource().getMemory() == 0) { queueInfo.setCapacity(0.0f); } else { queueInfo.setCapacity((float) getFairShare().getMemory() / scheduler.getClusterResource().getMemory()); } if (getFairShare().getMemory() == 0) { queueInfo.setCurrentCapacity(0.0f); } else { queueInfo.setCurrentCapacity((float) getResourceUsage().getMemory() / getFairShare().getMemory()); } ArrayList<QueueInfo> childQueueInfos = new ArrayList<QueueInfo>(); if (includeChildQueues) { Collection<FSQueue> childQueues = getChildQueues(); for (FSQueue child : childQueues) { childQueueInfos.add(child.getQueueInfo(recursive, recursive)); } } queueInfo.setChildQueues(childQueueInfos); queueInfo.setQueueState(QueueState.RUNNING); return queueInfo; }
while (cur != lowestCommonAncestor) { if (cur.getNumRunnableApps() == cur.getMaxRunningApps()) { throw new YarnException("Moving app attempt " + appAttId + " to queue " + queueName + " would violate queue maxRunningApps constraints on" + " queue " + cur.getQueueName()); if (!Resources.fitsIn(Resources.add(cur.getResourceUsage(), consumption), cur.getMaxShare())) { throw new YarnException("Moving app attempt " + appAttId + " to queue " + queueName + " would violate queue maxShare constraints on" + " queue " + cur.getQueueName()); cur = cur.getParent();
FSQueueMetrics queueMetrics = queue.getMetrics(); queueMetrics.setMinShare(queue.getMinShare()); queueMetrics.setMaxShare(queue.getMaxShare()); SchedulingPolicy policy = queueConf.getSchedulingPolicy(queue.getName()); policy.initialize(scheduler.getClusterResource()); queue.setPolicy(policy); } catch (AllocationConfigurationException ex) { LOG.warn("Cannot apply configured scheduling policy to queue " + queue.getName(), ex);
public Resource getHeadroom() { final FSQueue queue = (FSQueue) this.queue; SchedulingPolicy policy = queue.getPolicy(); Resource queueFairShare = queue.getFairShare(); Resource queueUsage = queue.getResourceUsage(); Resource clusterResource = this.scheduler.getClusterResource(); Resource clusterUsage = this.scheduler.getRootQueueMetrics() Resources.subtract(clusterResource, clusterUsage); Resource queueMaxAvailableResources = Resources.subtract(queue.getMaxShare(), queueUsage); Resource maxAvailableResource = Resources.componentwiseMin( clusterAvailableResources, queueMaxAvailableResources);
public FSQueue(String name, FairScheduler scheduler, FSParentQueue parent) { this.name = name; this.scheduler = scheduler; this.metrics = FSQueueMetrics.forQueue(getName(), parent, true, scheduler.getConf()); metrics.setMinShare(getMinShare()); metrics.setMaxShare(getMaxShare()); this.parent = parent; }
@Override public void updateDemand() { // Compute demand by iterating through apps in the queue // Limit demand to maxResources writeLock.lock(); try { demand = Resources.createResource(0); for (FSQueue childQueue : childQueues) { childQueue.updateDemand(); Resource toAdd = childQueue.getDemand(); demand = Resources.add(demand, toAdd); if (LOG.isDebugEnabled()) { LOG.debug("Counting resource from " + childQueue.getName() + " " + toAdd + "; Total resource demand for " + getName() + " now " + demand); } } // Cap demand to maxShare to limit allocation to maxShare demand = Resources.componentwiseMin(demand, getMaxShare()); } finally { writeLock.unlock(); } if (LOG.isDebugEnabled()) { LOG.debug("The updated demand for " + getName() + " is " + demand + "; the max is " + getMaxShare()); } }
static boolean fitsInMaxShare(FSQueue queue, Resource additionalResource) { Resource usagePlusAddition = Resources.add(queue.getResourceUsage(), additionalResource); if (!Resources.fitsIn(usagePlusAddition, queue.getMaxShare())) { return false; } FSQueue parentQueue = queue.getParent(); if (parentQueue != null) { return fitsInMaxShare(parentQueue, additionalResource); } return true; }
/** * Recompute the internal variables used by the scheduler - per-job weights, * fair shares, deficits, minimum slot allocations, and amount of used and * required resources per job. */ protected synchronized void update() { long start = getClock().getTime(); updateStarvationStats(); // Determine if any queues merit preemption FSQueue rootQueue = queueMgr.getRootQueue(); // Recursively update demands for all queues rootQueue.updateDemand(); rootQueue.setFairShare(clusterResource); // Recursively compute fair shares for all queues // and update metrics rootQueue.recomputeShares(); updateRootQueueMetrics(); if (LOG.isDebugEnabled()) { if (--updatesToSkipForDebug < 0) { updatesToSkipForDebug = UPDATE_DEBUG_FREQUENCY; LOG.debug("Cluster Capacity: " + clusterResource + " Allocations: " + rootMetrics.getAllocatedResources() + " Availability: " + Resource.newInstance( rootMetrics.getAvailableMB(), rootMetrics.getAvailableVirtualCores()) + " Demand: " + rootQueue.getDemand()); } } long duration = getClock().getTime() - start; fsOpDurations.addUpdateCallDuration(duration); }
/** * Recursively check policies for queues in pre-order. Get queue policies * from the allocation file instead of properties of {@link FSQueue} objects. * Set the policy for current queue if there is no policy violation for its * children. This method is invoked while reloading the allocation file. * * @param queueConf allocation configuration * @return true if no policy violation and successfully set polices * for queues; false otherwise */ public boolean verifyAndSetPolicyFromConf(AllocationConfiguration queueConf) { SchedulingPolicy queuePolicy = queueConf.getSchedulingPolicy(getName()); for (FSQueue child : getChildQueues()) { if (!queuePolicy.isChildPolicyAllowed( queueConf.getSchedulingPolicy(child.getName()))) { return false; } boolean success = child.verifyAndSetPolicyFromConf(queueConf); if (!success) { return false; } } // Set the policy if no policy violation for all children setPolicy(queuePolicy); return true; }
/** * Traverses the queue hierarchy under the given queue to gather all lists * of non-runnable applications. */ private void gatherPossiblyRunnableAppLists(FSQueue queue, List<List<FSAppAttempt>> appLists) { if (queue.getNumRunnableApps() < scheduler.getAllocationConfiguration() .getQueueMaxApps(queue.getName())) { if (queue instanceof FSLeafQueue) { appLists.add( ((FSLeafQueue)queue).getCopyOfNonRunnableAppSchedulables()); } else { for (FSQueue child : queue.getChildQueues()) { gatherPossiblyRunnableAppLists(child, appLists); } } } }
@Override public void setFairShare(Resource fairShare) { this.fairShare = fairShare; metrics.setFairShare(fairShare); if (LOG.isDebugEnabled()) { LOG.debug("The updated fairShare for " + getName() + " is " + fairShare); } }
@Override public void recomputeShares() { policy.computeShares(childQueues, getFairShare()); for (FSQueue childQueue : childQueues) { childQueue.getMetrics().setFairShare(childQueue.getFairShare()); childQueue.recomputeShares(); } }
/** * Helper method to check if the queue should attempt assigning resources * * @return true if check passes (can assign) or false otherwise */ protected boolean assignContainerPreCheck(FSSchedulerNode node) { if (!Resources.fitsIn(getResourceUsage(), scheduler.getAllocationConfiguration().getMaxResources(getName())) || node.getReservedContainer() != null) { return false; } return true; }