public synchronized void showRequests() { if (LOG.isDebugEnabled()) { for (Priority priority : getPriorities()) { Map<String, ResourceRequest> requests = getResourceRequests(priority); if (requests != null) { LOG.debug("showRequests:" + " application=" + getApplicationId() + " headRoom=" + getHeadroom() + " currentConsumption=" + currentConsumption.getMemory()); for (ResourceRequest request : requests.values()) { LOG.debug("showRequests:" + " application=" + getApplicationId() + " request=" + request); } } } } }
public SchedulerAppReport(SchedulerApplicationAttempt app) { this.live = app.getLiveContainers(); this.reserved = app.getReservedContainers(); this.pending = app.isPending(); }
protected void releaseContainers(List<ContainerId> containers, SchedulerApplicationAttempt attempt) { for (ContainerId containerId : containers) { RMContainer rmContainer = getRMContainer(containerId); if (rmContainer == null) { if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp() < nmExpireInterval) { LOG.info(containerId + " doesn't exist. Add the container" + " to the release request cache as it maybe on recovery."); attempt.getPendingRelease().add(containerId); } else { RMAuditLogger.logFailure(attempt.getUser(), AuditConstants.RELEASE_CONTAINER, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app or with invalid id.", attempt.getApplicationId(), containerId, null); } } completedContainer(rmContainer, SchedulerUtils.createAbnormalContainerStatus(containerId, SchedulerUtils.RELEASED_CONTAINER), RMContainerEventType.RELEASED); } }
public synchronized void transferStateFromPreviousAttempt( SchedulerApplicationAttempt appAttempt) { this.liveContainers = appAttempt.getLiveContainersMap(); // this.reReservations = appAttempt.reReservations; this.currentConsumption = appAttempt.getCurrentConsumption(); this.resourceLimit = appAttempt.getResourceLimit(); // this.currentReservation = appAttempt.currentReservation; // this.newlyAllocatedContainers = appAttempt.newlyAllocatedContainers; // this.schedulingOpportunities = appAttempt.schedulingOpportunities; this.lastScheduledContainer = appAttempt.getLastScheduledContainer(); this.appSchedulingInfo .transferStateFromPreviousAppSchedulingInfo(appAttempt.appSchedulingInfo); }
.getKeepContainersAcrossApplicationAttempts()) { if (schedulerAttempt.isStopped() || !schedulerAttempt.getApplicationAttemptId().equals( container.getContainerId().getApplicationAttemptId())) { LOG.info("Skip recovering container " + container Queue queue = schedulerAttempt.getQueue(); queue.recoverContainer(clusterResource, schedulerAttempt, rmContainer); schedulerAttempt.recoverContainer(rmContainer); Set<ContainerId> releases = schedulerAttempt.getPendingRelease(); if (releases.contains(container.getContainerId())) {
assertTrue(schedulerAttempt.getLiveContainers() .contains(scheduler.getRMContainer(amContainer.getContainerId()))); assertTrue(schedulerAttempt.getLiveContainers() .contains(scheduler.getRMContainer(runningContainer.getContainerId()))); assertEquals(schedulerAttempt.getCurrentConsumption(), usedResources); assertEquals((1L << 40) + 1L, schedulerAttempt.getNewContainerId());
@Override public FSLeafQueue getQueue() { return (FSLeafQueue)super.getQueue(); }
public void showRequests() { if (LOG.isDebugEnabled()) { try { readLock.lock(); for (SchedulerRequestKey schedulerKey : getSchedulerKeys()) { AppPlacementAllocator ap = getAppPlacementAllocator(schedulerKey); if (ap != null && ap.getOutstandingAsksCount(ResourceRequest.ANY) > 0) { LOG.debug("showRequests:" + " application=" + getApplicationId() + " headRoom=" + getHeadroom() + " currentConsumption=" + attemptResourceUsage.getUsed().getMemorySize()); ap.showRequests(); } } } finally { readLock.unlock(); } } }
Resource currentConsumption = applicationAttempt.getCurrentConsumption(); Assert.assertEquals(Resource.newInstance(0, 0), currentConsumption); Resource allocResources = applicationAttempt.getQueue().getMetrics().getAllocatedResources(); Assert.assertEquals(Resource.newInstance(0, 0), allocResources); .getLiveContainers(); Assert.assertEquals(3, liveContainers.size()); Iterator<RMContainer> iter = liveContainers.iterator(); currentConsumption = applicationAttempt.getCurrentConsumption(); Assert.assertEquals(Resource.newInstance(2048, 1), currentConsumption); allocResources = applicationAttempt.getQueue().getMetrics().getAllocatedResources(); Assert.assertEquals(Resource.newInstance(2048, 1), allocResources);
((AbstractYarnScheduler) rm1.getResourceScheduler()) .getCurrentAttemptForContainer(containerId6); while (schedulerAttempt.getReservedContainers().isEmpty()) { System.out.println("Waiting for container " + containerId6 + " to be reserved."); Assert.assertEquals(containerId6, schedulerAttempt.getReservedContainers() .get(0).getContainerId()); Assert.assertFalse(schedulerNewAttempt.getLiveContainers().contains( containerId2));
public RMContainer reserve(SchedulerNode node, SchedulerRequestKey schedulerKey, RMContainer rmContainer, Container container) { try { writeLock.lock(); // Create RMContainer if necessary if (rmContainer == null) { rmContainer = new RMContainerImpl(container, schedulerKey, getApplicationAttemptId(), node.getNodeID(), appSchedulingInfo.getUser(), rmContext); } if (rmContainer.getState() == RMContainerState.NEW) { attemptResourceUsage.incReserved(node.getPartition(), container.getResource()); ((RMContainerImpl) rmContainer).setQueueName(this.getQueueName()); // Reset the re-reservation count resetReReservations(schedulerKey); } else{ // Note down the re-reservation addReReservation(schedulerKey); } commonReserve(node, schedulerKey, rmContainer, container.getResource()); return rmContainer; } finally { writeLock.unlock(); } }
@VisibleForTesting public void clearPendingContainerCache() { for (SchedulerApplication<T> app : applications.values()) { T attempt = app.getCurrentAppAttempt(); if (attempt != null) { for (ContainerId containerId : attempt.getPendingRelease()) { RMAuditLogger.logFailure(app.getUser(), AuditConstants.RELEASE_CONTAINER, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app " + "or with invalid id.", attempt.getApplicationId(), containerId, null); } attempt.getPendingRelease().clear(); } } }
synchronized void allocateResource(Resource clusterResource, SchedulerApplicationAttempt application, Resource resource, Set<String> nodeLabels) { super.allocateResource(clusterResource, resource, nodeLabels); // Update user metrics String userName = application.getUser(); User user = getUser(userName); user.assignContainer(resource, nodeLabels); // Note this is a bit unconventional since it gets the object and modifies // it here, rather then using set routine Resources.subtractFrom(application.getHeadroom(), resource); // headroom metrics.setAvailableResourcesToUser(userName, application.getHeadroom()); if (LOG.isDebugEnabled()) { LOG.info(getQueueName() + " user=" + userName + " used=" + queueUsage.getUsed() + " numContainers=" + numContainers + " headroom = " + application.getHeadroom() + " user-resources=" + user.getUsed() ); } }
new RMContainerImpl(container, getApplicationAttemptId(), node.getNodeID(), appSchedulingInfo.getUser(), rmContext); resetReReservations(priority); } else { addReReservation(priority); LOG.debug("Application attempt " + getApplicationAttemptId() + " reserved container " + rmContainer + " on node " + node + ". This attempt currently has " + reservedContainers.size()
public static void waitForNumContainersToRecover(int num, MockRM rm, ApplicationAttemptId attemptId) throws Exception { AbstractYarnScheduler scheduler = (AbstractYarnScheduler) rm.getResourceScheduler(); SchedulerApplicationAttempt attempt = scheduler.getApplicationAttempt(attemptId); while (attempt == null) { System.out.println("Wait for scheduler attempt " + attemptId + " to be created"); Thread.sleep(200); attempt = scheduler.getApplicationAttempt(attemptId); } while (attempt.getLiveContainers().size() < num) { System.out.println("Wait for " + num + " containers to recover. currently: " + attempt.getLiveContainers().size()); Thread.sleep(200); } }
@Override public long generateContainerId() { return appAttempt.getAppSchedulingInfo().getNewContainerId(); } });
resource, Resources.none())) { ContainerId cId = ContainerId.newContainerId(appAttempt.getApplicationAttemptId(), appAttempt.getAppSchedulingInfo().getNewContainerId()); Container container = BuilderUtils.newContainer( cId, schedulerNode.getNodeID(), schedulerNode.getHttpAddress(), RMContainer rmContainer = new RMContainerImpl(container, SchedulerRequestKey.extractFrom(container), appAttempt.getApplicationAttemptId(), container.getNodeId(), appAttempt.getUser(), rmContext, false); ((RMContainerImpl)rmContainer).setAllocationTags( new HashSet<>(schedulingRequest.getAllocationTags()));
@Override public Resource getHeadroom() { try { readLock.lock(); if (headroomProvider != null) { return headroomProvider.getHeadroom(); } return super.getHeadroom(); } finally { readLock.unlock(); } }
public Boolean get() { // release cache is cleaned up and previous running container is not // recovered return scheduler.getApplicationAttempt(am1.getApplicationAttemptId()) .getPendingRelease().isEmpty() && scheduler.getRMContainer(runningContainer) == null; } }, 1000, 20000);
@Test public void testAppPercentagesOnswitch() throws Exception { FifoScheduler scheduler = mock(FifoScheduler.class); when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(0, 0)); when(scheduler.getResourceCalculator()) .thenReturn(new DefaultResourceCalculator()); ApplicationAttemptId appAttId = createAppAttemptId(0, 0); RMContext rmContext = mock(RMContext.class); when(rmContext.getEpoch()).thenReturn(3L); when(rmContext.getScheduler()).thenReturn(scheduler); final String user = "user1"; Queue queue = createQueue("test", null); SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, user, queue, queue.getAbstractUsersManager(), rmContext); // Resource request Resource requestedResource = Resource.newInstance(1536, 2); app.attemptResourceUsage.incUsed(requestedResource); assertEquals(0.0f, app.getResourceUsageReport().getQueueUsagePercentage(), 0.0f); assertEquals(0.0f, app.getResourceUsageReport().getClusterUsagePercentage(), 0.0f); }