CSAssignment assignment = new CSAssignment(Resources.createResource(0, 0), NodeType.NODE_LOCAL); assignment.setType(assignedToChild.getType()); assignment.setRequestLocalityType( assignedToChild.getRequestLocalityType()); assignment.setExcessReservation(assignedToChild.getExcessReservation()); assignment.setContainersToKill(assignedToChild.getContainersToKill()); assignedToChild.getResource(), Resources.none())) { assignedToChild.getAssignmentInformation().getReservationDetails() != null && !assignedToChild.getAssignmentInformation() .getReservationDetails().isEmpty(); if (node != null && !isReserved) { ActivitiesLogger.NODE.finishAllocatedNodeAllocation( activitiesManager, node, assignedToChild.getAssignmentInformation() .getFirstAllocatedOrReservedContainerId(), AllocationState.ALLOCATED); ActivitiesLogger.NODE.finishAllocatedNodeAllocation( activitiesManager, node, assignedToChild.getAssignmentInformation() .getFirstAllocatedOrReservedContainerId(), AllocationState.RESERVED); Resources.addTo(assignment.getResource(),
private List<SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode>> getSchedulerContainersToRelease( CSAssignment csAssignment) { List<SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode>> list = null; if (csAssignment.getContainersToKill() != null && !csAssignment .getContainersToKill().isEmpty()) { list = new ArrayList<>(); for (RMContainer rmContainer : csAssignment.getContainersToKill()) { list.add(getSchedulerContainer(rmContainer, false)); } } if (csAssignment.getExcessReservation() != null) { if (null == list) { list = new ArrayList<>(); } list.add( getSchedulerContainer(csAssignment.getExcessReservation(), false)); } return list; }
private boolean canAllocateMore(CSAssignment assignment, int offswitchCount, int assignedContainers) { // Current assignment shouldn't be empty if (assignment == null || Resources.equals(assignment.getResource(), Resources.none())) { return false; } // offswitch assignment should be under threshold if (offswitchCount >= offswitchPerHeartbeatLimit) { return false; } // And it should not be a reserved container if (assignment.getAssignmentInformation().getNumReservations() > 0) { return false; } // assignMultipleEnabled should be ON, // and assignedContainers should be under threshold return assignMultipleEnabled && (maxAssignPerHeartbeat == -1 || assignedContainers < maxAssignPerHeartbeat); }
csAssignment.getResource(), Resources.none())) { csAssignment.getAssignmentInformation().getAllocationDetails(); if (!allocations.isEmpty()) { RMContainer rmContainer = allocations.get(0).rmContainer; getSchedulerContainer(rmContainer, true), getSchedulerContainersToRelease(csAssignment), getSchedulerContainer(csAssignment.getFulfilledReservedContainer(), false), csAssignment.getType(), csAssignment.getRequestLocalityType(), csAssignment.getSchedulingMode() != null ? csAssignment.getSchedulingMode() : SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY, csAssignment.getResource()); csAssignment.getAssignmentInformation().getReservationDetails(); if (!reservation.isEmpty()) { RMContainer rmContainer = reservation.get(0).rmContainer; getSchedulerContainer(rmContainer, false), getSchedulerContainersToRelease(csAssignment), getSchedulerContainer(csAssignment.getFulfilledReservedContainer(), false), csAssignment.getType(), csAssignment.getRequestLocalityType(), csAssignment.getSchedulingMode() != null ? csAssignment.getSchedulingMode() : SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY, csAssignment.getResource());
CSAssignment.SkippedType.OTHER : CSAssignment.SkippedType.NONE; CSAssignment assignment = new CSAssignment(skipped); assignment.setApplication(application); assignment.setExcessReservation(result.getContainerToBeUnreserved()); assignment.setRequestLocalityType(result.requestLocalityType); assignment.setResource(allocatedResource); assignment.setType(result.getContainerNodeType()); assignment.getAssignmentInformation().addReservationDetails( updatedContainer, application.getCSLeafQueue().getQueuePath()); assignment.getAssignmentInformation().incrReservations(); Resources.addTo(assignment.getAssignmentInformation().getReserved(), allocatedResource); .getContainerId() + " queue=" + appInfo.getQueueName() + " clusterResource=" + clusterResource + " type=" + assignment.getType() + " requestedPartition=" + updatedContainer.getNodeLabelExpression()); assignment.getAssignmentInformation().addAllocationDetails( updatedContainer, application.getCSLeafQueue().getQueuePath()); assignment.getAssignmentInformation().incrAllocations(); Resources.addTo(assignment.getAssignmentInformation().getAllocated(), allocatedResource);
FiCaSchedulerNode node, ResourceLimits resourceLimits) { CSAssignment assignment = new CSAssignment(Resources.createResource(0, 0), NodeType.NODE_LOCAL); Set<String> nodeLabels = node.getLabels(); assignment.setType(assignedToChild.getType()); assignedToChild.getResource(), Resources.none())) { super.allocateResource(clusterResource, assignedToChild.getResource(), nodeLabels); Resources.addTo(assignment.getResource(), assignedToChild.getResource()); + " assignedSoFarInThisIteration=" + assignment.getResource() + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity()); if (!rootQueue || assignment.getType() == NodeType.OFF_SWITCH) { if (LOG.isDebugEnabled()) { if (rootQueue && assignment.getType() == NodeType.OFF_SWITCH) { LOG.debug("Not assigning more than one off-switch container," + " assignments so far: " + assignment);
private synchronized CSAssignment assignContainersToChildQueues( Resource cluster, FiCaSchedulerNode node, ResourceLimits limits) { CSAssignment assignment = new CSAssignment(Resources.createResource(0, 0), NodeType.NODE_LOCAL); LOG.debug("Assigned to queue: " + childQueue.getQueuePath() + " stats: " + childQueue + " --> " + assignment.getResource() + ", " + assignment.getType()); assignment.getResource(), Resources.none())) {
if (assignment.getSkipped()) { Resource assigned = assignment.getResource(); if (Resources.greaterThan( resourceCalculator, clusterResource, assigned, Resources.none())) { if (assignment.getType() != NodeType.OFF_SWITCH) { if (LOG.isDebugEnabled()) { LOG.debug("Resetting scheduling opportunities"); if (assignment.getType() == NodeType.NODE_LOCAL || getRackLocalityFullReset()) { application.resetSchedulingOpportunities(priority);
private synchronized CSAssignment assignReservedContainer( FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer rmContainer, Resource clusterResource) { // Do we still need this reservation? Priority priority = rmContainer.getReservedPriority(); if (application.getTotalRequiredResources(priority) == 0) { // Release return new CSAssignment(application, rmContainer); } // Try to assign if we have sufficient resources assignContainersOnNode(clusterResource, node, application, priority, rmContainer, new ResourceLimits(Resources.none())); // Doesn't matter... since it's already charged for at time of reservation // "re-reservation" is *free* return new CSAssignment(Resources.none(), NodeType.NODE_LOCAL); }
private void verifyContainerAllocated(CSAssignment assignment, NodeType nodeType) { Assert.assertTrue(Resources.greaterThan(resourceCalculator, null, assignment.getResource(), Resources.none())); Assert .assertTrue(assignment.getAssignmentInformation().getNumAllocations() > 0); Assert.assertEquals(nodeType, assignment.getType()); }
LOG.debug("Assigned to queue: " + childQueue.getQueuePath() + " stats: " + childQueue + " --> " + childAssignment.getResource() + ", " + childAssignment.getType()); childAssignment.getResource(), Resources.none())) { assignment = childAssignment; break; } else if (childAssignment.getSkippedType() == CSAssignment.SkippedType.QUEUE_LIMIT) { if (assignment.getSkippedType() != CSAssignment.SkippedType.QUEUE_LIMIT) { assignment = childAssignment;
if (assignment.getType() == NodeType.OFF_SWITCH) { offswitchCount++; assignment.getResource(), Resources.none())) { assignedContainers++; && assignment.getType() == NodeType.OFF_SWITCH) { offswitchCount++; assignment.getResource(), Resources.none())) { assignedContainers++;
assertEquals(1, app_0.getSchedulingOpportunities(priority)); assertEquals(3, app_0.getTotalRequiredResources(priority)); assertEquals(NodeType.NODE_LOCAL, assignment.getType()); // None->NODE_LOCAL assertEquals(2, app_0.getSchedulingOpportunities(priority)); assertEquals(3, app_0.getTotalRequiredResources(priority)); assertEquals(NodeType.NODE_LOCAL, assignment.getType()); // None->NODE_LOCAL assertEquals(3, app_0.getSchedulingOpportunities(priority)); assertEquals(3, app_0.getTotalRequiredResources(priority)); assertEquals(NodeType.NODE_LOCAL, assignment.getType()); // None->NODE_LOCAL assertEquals(4, app_0.getSchedulingOpportunities(priority)); // should NOT reset assertEquals(2, app_0.getTotalRequiredResources(priority)); assertEquals(NodeType.OFF_SWITCH, assignment.getType()); assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should reset assertEquals(1, app_0.getTotalRequiredResources(priority)); assertEquals(NodeType.NODE_LOCAL, assignment.getType()); assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should reset assertEquals(0, app_0.getTotalRequiredResources(priority)); assertEquals(NodeType.NODE_LOCAL, assignment.getType()); assertEquals(1, app_0.getSchedulingOpportunities(priority)); assertEquals(4, app_0.getTotalRequiredResources(priority)); assertEquals(NodeType.NODE_LOCAL, assignment.getType()); // None->NODE_LOCAL assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should reset
RMNodeLabelsManager.NO_LABEL, clusterResource))); RMContainer excessReservation = assignment.getExcessReservation(); if (excessReservation != null) { Container container = excessReservation.getContainer(); queue.completedContainer( clusterResource, assignment.getApplication(), node, excessReservation, SchedulerUtils.createAbnormalContainerStatus(
b.assignContainers(clusterResource, node0, new ResourceLimits( clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); LOG.info(assign.toString()); applyCSAssignment(clusterResource, assign, b, nodes, apps); } while (assign.getResource().getMemorySize() > 0 && assign.getAssignmentInformation().getNumReservations() == 0); do { assign = clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); applyCSAssignment(clusterResource, assign, b, nodes, apps); } while (assign.getResource().getMemorySize() > 0 && assign.getAssignmentInformation().getNumReservations() == 0);
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); assignment.setSchedulingMode(SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); submitResourceCommitRequest(getClusterResource(), assignment); assignment.getResource(), Resources.none())) { FiCaSchedulerNode node = CandidateNodeSetUtils.getSingleNode(candidates); NodeId nodeId = null; getClusterResource())), SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY); assignment.setSchedulingMode(SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY); submitResourceCommitRequest(getClusterResource(), assignment);
Resource assigned = assignment.getResource(); ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY); return assignment; } else if (assignment.getSkippedType() == CSAssignment.SkippedType.OTHER) { ActivitiesLogger.APP.finishSkippedAppAllocationRecording( ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY); application.updateNodeInfoForAMDiagnostics(node); } else if (assignment.getSkippedType() == CSAssignment.SkippedType.QUEUE_LIMIT) { return assignment;
clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY); assertTrue("Still within limits, should assign", assign.getResource().getMemorySize() > 0);
assertEquals(0*GB, node_0.getUsedResource().getMemory()); assertEquals(4*GB, assignment.getExcessReservation().getContainer().getResource().getMemory());
private void updateSchedulerHealth(long now, NodeId nodeId, CSAssignment assignment) { List<AssignmentInformation.AssignmentDetails> allocations = assignment.getAssignmentInformation().getAllocationDetails(); List<AssignmentInformation.AssignmentDetails> reservations = assignment.getAssignmentInformation().getReservationDetails(); .getAssignmentInformation().getNumReservations()); schedulerHealth.updateSchedulerAllocationCounts(assignment .getAssignmentInformation().getNumAllocations()); schedulerHealth.updateSchedulerRunDetails(now, assignment .getAssignmentInformation().getAllocated(), assignment .getAssignmentInformation().getReserved());