@Override public void incUsedResource(String nodeLabel, Resource resourceToInc, SchedulerApplicationAttempt application) { usersManager.updateUserResourceUsage(application.getUser(), resourceToInc, nodeLabel, true); super.incUsedResource(nodeLabel, resourceToInc, application); }
@Override public void decUsedResource(String nodeLabel, Resource resourceToDec, SchedulerApplicationAttempt application) { usersManager.updateUserResourceUsage(application.getUser(), resourceToDec, nodeLabel, false); super.decUsedResource(nodeLabel, resourceToDec, application); }
private void updateNMToken(Container container) { NMToken nmToken = rmContext.getNMTokenSecretManager().createAndGetNMToken(getUser(), getApplicationAttemptId(), container); if (nmToken != null) { updatedNMTokens.add(nmToken); } }
public void decAMUsedResource(String nodeLabel, Resource resourceToDec, SchedulerApplicationAttempt application) { getUser(application.getUser()).getResourceUsage().decAMUsed(nodeLabel, resourceToDec); // ResourceUsage has its own lock, no addition lock needs here. queueUsage.decAMUsed(nodeLabel, resourceToDec); }
public void incAMUsedResource(String nodeLabel, Resource resourceToInc, SchedulerApplicationAttempt application) { getUser(application.getUser()).getResourceUsage().incAMUsed(nodeLabel, resourceToInc); // ResourceUsage has its own lock, no addition lock needs here. queueUsage.incAMUsed(nodeLabel, resourceToInc); }
String userName = application.getUser();
protected void releaseContainers(List<ContainerId> containers, SchedulerApplicationAttempt attempt) { for (ContainerId containerId : containers) { RMContainer rmContainer = getRMContainer(containerId); if (rmContainer == null) { if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp() < nmExpireInterval) { LOG.info(containerId + " doesn't exist. Add the container" + " to the release request cache as it maybe on recovery."); attempt.getPendingRelease().add(containerId); } else { RMAuditLogger.logFailure(attempt.getUser(), AuditConstants.RELEASE_CONTAINER, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app or with invalid id.", attempt.getApplicationId(), containerId, null); } } completedContainer(rmContainer, SchedulerUtils.createAbnormalContainerStatus(containerId, SchedulerUtils.RELEASED_CONTAINER), RMContainerEventType.RELEASED); } }
protected void releaseContainers(List<ContainerId> containers, SchedulerApplicationAttempt attempt) { for (ContainerId containerId : containers) { RMContainer rmContainer = getRMContainer(containerId); if (rmContainer == null) { if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp() < nmExpireInterval) { LOG.info(containerId + " doesn't exist. Add the container" + " to the release request cache as it maybe on recovery."); synchronized (attempt) { attempt.getPendingRelease().add(containerId); } } else { RMAuditLogger.logFailure(attempt.getUser(), AuditConstants.RELEASE_CONTAINER, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app or with invalid id.", attempt.getApplicationId(), containerId); } } completedContainer(rmContainer, SchedulerUtils.createAbnormalContainerStatus(containerId, SchedulerUtils.RELEASED_CONTAINER), RMContainerEventType.RELEASED); } }
protected void releaseContainers(List<ContainerId> containers, SchedulerApplicationAttempt attempt) { for (ContainerId containerId : containers) { RMContainer rmContainer = getRMContainer(containerId); if (rmContainer == null) { if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp() < nmExpireInterval) { LOG.info(containerId + " doesn't exist. Add the container" + " to the release request cache as it maybe on recovery."); synchronized (attempt) { attempt.getPendingRelease().add(containerId); } } else { RMAuditLogger.logFailure(attempt.getUser(), AuditConstants.RELEASE_CONTAINER, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app or with invalid id.", attempt.getApplicationId(), containerId); } } completedContainer(rmContainer, SchedulerUtils.createAbnormalContainerStatus(containerId, SchedulerUtils.RELEASED_CONTAINER), RMContainerEventType.RELEASED); } }
public synchronized void move(Queue newQueue) { QueueMetrics oldMetrics = queue.getMetrics(); QueueMetrics newMetrics = newQueue.getMetrics(); String user = getUser(); for (RMContainer liveContainer : liveContainers.values()) { Resource resource = liveContainer.getContainer().getResource(); oldMetrics.releaseResources(user, 1, resource); newMetrics.allocateResources(user, 1, resource, false); } for (Map<NodeId, RMContainer> map : reservedContainers.values()) { for (RMContainer reservedContainer : map.values()) { Resource resource = reservedContainer.getReservedResource(); oldMetrics.unreserveResource(user, resource); newMetrics.reserveResource(user, resource); } } appSchedulingInfo.move(newQueue); this.queue = newQueue; }
public synchronized void move(Queue newQueue) { QueueMetrics oldMetrics = queue.getMetrics(); QueueMetrics newMetrics = newQueue.getMetrics(); String user = getUser(); for (RMContainer liveContainer : liveContainers.values()) { Resource resource = liveContainer.getContainer().getResource(); oldMetrics.releaseResources(user, 1, resource); newMetrics.allocateResources(user, 1, resource, false); } for (Map<NodeId, RMContainer> map : reservedContainers.values()) { for (RMContainer reservedContainer : map.values()) { Resource resource = reservedContainer.getReservedResource(); oldMetrics.unreserveResource(user, resource); newMetrics.reserveResource(user, resource); } } appSchedulingInfo.move(newQueue); this.queue = newQueue; }
synchronized void allocateResource(Resource clusterResource, SchedulerApplicationAttempt application, Resource resource, Set<String> nodeLabels) { super.allocateResource(clusterResource, resource, nodeLabels); // Update user metrics String userName = application.getUser(); User user = getUser(userName); user.assignContainer(resource, nodeLabels); // Note this is a bit unconventional since it gets the object and modifies // it here, rather then using set routine Resources.subtractFrom(application.getHeadroom(), resource); // headroom metrics.setAvailableResourcesToUser(userName, application.getHeadroom()); if (LOG.isDebugEnabled()) { LOG.info(getQueueName() + " user=" + userName + " used=" + queueUsage.getUsed() + " numContainers=" + numContainers + " headroom = " + application.getHeadroom() + " user-resources=" + user.getUsed() ); } }
synchronized void allocateResource(Resource clusterResource, SchedulerApplicationAttempt application, Resource resource, Set<String> nodeLabels) { super.allocateResource(clusterResource, resource, nodeLabels); // Update user metrics String userName = application.getUser(); User user = getUser(userName); user.assignContainer(resource, nodeLabels); // Note this is a bit unconventional since it gets the object and modifies // it here, rather then using set routine Resources.subtractFrom(application.getHeadroom(), resource); // headroom metrics.setAvailableResourcesToUser(userName, application.getHeadroom()); if (LOG.isDebugEnabled()) { LOG.info(getQueueName() + " user=" + userName + " used=" + queueUsage.getUsed() + " numContainers=" + numContainers + " headroom = " + application.getHeadroom() + " user-resources=" + user.getUsed() ); } }
QueueMetrics newMetrics = newQueue.getMetrics(); String newQueueName = newQueue.getQueueName(); String user = getUser();
SchedulerRequestKey.extractFrom(container), appAttempt.getApplicationAttemptId(), container.getNodeId(), appAttempt.getUser(), rmContext, false); ((RMContainerImpl)rmContainer).setAllocationTags( new HashSet<>(schedulingRequest.getAllocationTags()));
public static RMContainer createOpportunisticRmContainer(RMContext rmContext, Container container, boolean isRemotelyAllocated) { SchedulerApplicationAttempt appAttempt = ((AbstractYarnScheduler) rmContext.getScheduler()) .getCurrentAttemptForContainer(container.getId()); RMContainer rmContainer = new RMContainerImpl(container, SchedulerRequestKey.extractFrom(container), appAttempt.getApplicationAttemptId(), container.getNodeId(), appAttempt.getUser(), rmContext, isRemotelyAllocated); appAttempt.addRMContainer(container.getId(), rmContainer); ((AbstractYarnScheduler) rmContext.getScheduler()).getNode( container.getNodeId()).allocateContainer(rmContainer); return rmContainer; } }
request.getResourceBlacklistRequest(), partitionedAsks.getOpportunistic(), appAttemptId, oppCtx, ResourceManager.getClusterTimeStamp(), appAttempt.getUser());
private RMContainer createDecreasedRMContainer( SchedulerApplicationAttempt appAttempt, UpdateContainerRequest uReq, RMContainer rmContainer) { SchedulerRequestKey sk = SchedulerRequestKey.extractFrom(rmContainer.getContainer()); Container decreasedContainer = BuilderUtils.newContainer( ContainerId.newContainerId(appAttempt.getApplicationAttemptId(), appAttempt.getNewContainerId()), rmContainer.getContainer().getNodeId(), rmContainer.getContainer().getNodeHttpAddress(), Resources.none(), sk.getPriority(), null, rmContainer.getExecutionType(), sk.getAllocationRequestId()); decreasedContainer.setVersion(rmContainer.getContainer().getVersion()); RMContainer newRmContainer = new RMContainerImpl(decreasedContainer, sk, appAttempt.getApplicationAttemptId(), decreasedContainer.getNodeId(), appAttempt.getUser(), rmContext, rmContainer.isRemotelyAllocated()); appAttempt.addRMContainer(decreasedContainer.getId(), rmContainer); ((AbstractYarnScheduler) rmContext.getScheduler()).getNode( decreasedContainer.getNodeId()).allocateContainer(newRmContainer); return newRmContainer; }