private List<MaxResourceValidationResult> validateResourceRequests( List<ResourceRequest> requests, FSLeafQueue queue) { List<MaxResourceValidationResult> validationResults = Lists.newArrayList(); for (ResourceRequest resourceRequest : requests) { if (LOG.isTraceEnabled()) { LOG.trace("Validating resource request: " + resourceRequest); } MaxResourceValidationResult validationResult = SchedulerUtils.validateResourceRequestsAgainstQueueMaxResource( resourceRequest, queue.getMaxShare()); if (!validationResult.isValid()) { validationResults.add(validationResult); LOG.warn(String.format("Queue %s cannot handle resource request" + "because it has zero available amount of resource " + "for a requested resource type, " + "so the resource request is ignored!" + " Requested resources: %s, " + "maximum queue resources: %s", queue.getName(), resourceRequest.getCapability(), queue.getMaxShare())); } } return validationResults; }
@Override public void updateDemand() { // Compute demand by iterating through apps in the queue // Limit demand to maxResources Resource tmpDemand = Resources.createResource(0); readLock.lock(); try { for (FSAppAttempt sched : runnableApps) { sched.updateDemand(); Resources.addTo(tmpDemand, sched.getDemand()); } for (FSAppAttempt sched : nonRunnableApps) { sched.updateDemand(); Resources.addTo(tmpDemand, sched.getDemand()); } } finally { readLock.unlock(); } // Cap demand to maxShare to limit allocation to maxShare demand = Resources.componentwiseMin(tmpDemand, getMaxShare()); if (LOG.isDebugEnabled()) { LOG.debug("The updated demand for " + getName() + " is " + demand + "; the max is " + getMaxShare()); LOG.debug("The updated fairshare for " + getName() + " is " + getFairShare()); } }
/** * Compute the maximum resource AM can use. The value is the result of * multiplying FairShare and maxAMShare. If FairShare is zero, use * min(maxShare, available resource) instead to prevent zero value for * maximum AM resource since it forbids any job running in the queue. * * @return the maximum resource AM can use */ private Resource computeMaxAMResource() { Resource maxResource = Resources.clone(getFairShare()); if (maxResource.getMemorySize() == 0) { maxResource.setMemorySize( Math.min(scheduler.getRootQueueMetrics().getAvailableMB(), getMaxShare().getMemorySize())); } if (maxResource.getVirtualCores() == 0) { maxResource.setVirtualCores(Math.min( scheduler.getRootQueueMetrics().getAvailableVirtualCores(), getMaxShare().getVirtualCores())); } // Round up to allow AM to run when there is only one vcore on the cluster return Resources.multiplyAndRoundUp(maxResource, maxAMShare); }
+ "maximum queue resources: %s", applicationId, queue.getName(), invalidAMResourceRequests, queue.getMaxShare()); rejectApplicationWithMessage(applicationId, msg); queue.removeAssignedApp(applicationId);
@Override protected void dumpStateInternal(StringBuilder sb) { sb.append("{Name: " + getName() + ", Weight: " + weights + ", Policy: " + policy.getName() + ", FairShare: " + getFairShare() + ", SteadyFairShare: " + getSteadyFairShare() + ", MaxShare: " + getMaxShare() + ", MinShare: " + minShare + ", ResourceUsage: " + getResourceUsage() + ", Demand: " + getDemand() + ", Runnable: " + getNumRunnableApps() + ", NumPendingApps: " + getNumPendingApps() + ", NonRunnable: " + getNumNonRunnableApps() + ", MaxAMShare: " + maxAMShare + ", MaxAMResource: " + computeMaxAMResource() + ", AMResourceUsage: " + getAmResourceUsage() + ", LastTimeAtMinShare: " + lastTimeAtMinShare + "}"); }
Mockito.when(mockQueue.getMaxShare()).thenReturn(queueMaxResources); Mockito.when(mockQueue.getFairShare()).thenReturn(queueFairShare); Mockito.when(mockQueue.getResourceUsage()).thenReturn(queueUsage);
Mockito.when(mockQueue.getMaxShare()).thenReturn(queueMaxResources); Mockito.when(mockQueue.getFairShare()).thenReturn(queueFairShare); Mockito.when(mockQueue.getResourceUsage()).thenReturn(queueUsage);
"Max share of queue: " + getQueue().getMaxShare());
0, queueFSZeroWithMax.getFairShare().getMemorySize()); Resource expectedAMResource = Resources.multiplyAndRoundUp( queueFSZeroWithMax.getMaxShare(), queueFSZeroWithMax.getMaxAMShare()); assertEquals("QueueFSZeroWithMax's maximum AM resource should be " + "maxShare * maxAMShare", expectedAMResource.getMemorySize(),
assertEquals("Max resources for root.test.childA were inherited from " + "parent's max child resources", Resources.unbounded(), queueManager.getLeafQueue("root.test.childA", false).getMaxShare()); assertNotNull("Leaf queue root.test.childB was not created during setup", queueManager.getParentQueue("root.test.childB", false));