private boolean hasRequestLabelChanged(ResourceRequest requestOne, ResourceRequest requestTwo) { String requestOneLabelExp = requestOne.getNodeLabelExpression(); String requestTwoLabelExp = requestTwo.getNodeLabelExpression(); // First request label expression can be null and second request // is not null then we have to consider it as changed. if ((null == requestOneLabelExp) && (null != requestTwoLabelExp)) { return true; } // If the label is not matching between both request when // requestOneLabelExp is not null. return ((null != requestOneLabelExp) && !(requestOneLabelExp .equals(requestTwoLabelExp))); }
private boolean isRequestLabelChanged(ResourceRequest requestOne, ResourceRequest requestTwo) { String requestOneLabelExp = requestOne.getNodeLabelExpression(); String requestTwoLabelExp = requestTwo.getNodeLabelExpression(); // First request label expression can be null and second request // is not null then we have to consider it as changed. if ((null == requestOneLabelExp) && (null != requestTwoLabelExp)) { return true; } // If the label is not matching between both request when // requestOneLabelExp is not null. return ((null != requestOneLabelExp) && !(requestOneLabelExp .equals(requestTwoLabelExp))); }
@Override public String getAmNodeLabelExpression() { String amNodeLabelExpression = null; if (!getApplicationSubmissionContext().getUnmanagedAM()) { amNodeLabelExpression = getAMResourceRequests() != null && !getAMResourceRequests().isEmpty() ? getAMResourceRequests().get(0).getNodeLabelExpression() : null; amNodeLabelExpression = (amNodeLabelExpression == null) ? NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET : amNodeLabelExpression; amNodeLabelExpression = (amNodeLabelExpression.trim().isEmpty()) ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : amNodeLabelExpression; } return amNodeLabelExpression; }
private static void normalizeNodeLabelExpressionInRequest( ResourceRequest resReq, QueueInfo queueInfo) { String labelExp = resReq.getNodeLabelExpression(); // if queue has default label expression, and RR doesn't have, use the // default label expression of queue if (labelExp == null && queueInfo != null && ResourceRequest.ANY .equals(resReq.getResourceName())) { labelExp = queueInfo.getDefaultNodeLabelExpression(); } // If labelExp still equals to null, set it to be NO_LABEL if (labelExp == null) { labelExp = RMNodeLabelsManager.NO_LABEL; } resReq.setNodeLabelExpression(labelExp); }
private static void normalizeNodeLabelExpressionInRequest( ResourceRequest resReq, QueueInfo queueInfo) { String labelExp = resReq.getNodeLabelExpression(); // if queue has default label expression, and RR doesn't have, use the // default label expression of queue if (labelExp == null && queueInfo != null && ResourceRequest.ANY .equals(resReq.getResourceName())) { labelExp = queueInfo.getDefaultNodeLabelExpression(); } // If labelExp still equals to null, set it to be NO_LABEL if (labelExp == null) { labelExp = RMNodeLabelsManager.NO_LABEL; } resReq.setNodeLabelExpression(labelExp); }
private List<ResourceRequest> cloneAsks() { List<ResourceRequest> askList = new ArrayList<ResourceRequest>(ask.size()); for(ResourceRequest r : ask) { // create a copy of ResourceRequest as we might change it while the // RPC layer is using it to send info across ResourceRequest rr = ResourceRequest.newInstance(r.getPriority(), r.getResourceName(), r.getCapability(), r.getNumContainers(), r.getRelaxLocality(), r.getNodeLabelExpression()); askList.add(rr); } return askList; }
private void updateNodeLabels(ResourceRequest request) { String resourceName = request.getResourceName(); if (resourceName.equals(ResourceRequest.ANY)) { ResourceRequest previousAnyRequest = getResourceRequest(resourceName); // When there is change in ANY request label expression, we should // update label for all resource requests already added of same // priority as ANY resource request. if ((null == previousAnyRequest) || hasRequestLabelChanged( previousAnyRequest, request)) { for (ResourceRequest r : resourceRequestMap.values()) { if (!r.getResourceName().equals(ResourceRequest.ANY)) { r.setNodeLabelExpression(request.getNodeLabelExpression()); } } } } else{ ResourceRequest anyRequest = getResourceRequest(ResourceRequest.ANY); if (anyRequest != null) { request.setNodeLabelExpression(anyRequest.getNodeLabelExpression()); } } }
private void decrementOutstanding(SchedulerRequestKey schedulerRequestKey, ResourceRequest offSwitchRequest) { int numOffSwitchContainers = offSwitchRequest.getNumContainers() - 1; offSwitchRequest.setNumContainers(numOffSwitchContainers); // Do we have any outstanding requests? // If there is nothing, we need to deactivate this application if (numOffSwitchContainers == 0) { appSchedulingInfo.getSchedulerKeys().remove(schedulerRequestKey); appSchedulingInfo.checkForDeactivation(); resourceRequestMap.remove(ResourceRequest.ANY); if (resourceRequestMap.isEmpty()) { appSchedulingInfo.removeAppPlacement(schedulerRequestKey); } } appSchedulingInfo.decPendingResource( offSwitchRequest.getNodeLabelExpression(), offSwitchRequest.getCapability()); }
public RMApp submitApp(List<ResourceRequest> amResourceRequests) throws Exception { return submitApp(amResourceRequests, "app1", "user", null, false, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, false, null, 0, null, true, amResourceRequests.get(0).getPriority(), amResourceRequests.get(0).getNodeLabelExpression(), null, null); }
public RMApp submitApp(List<ResourceRequest> amResourceRequests, String appNodeLabel) throws Exception { return submitApp(amResourceRequests, "app1", "user", null, false, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, false, null, 0, null, true, amResourceRequests.get(0).getPriority(), amResourceRequests.get(0).getNodeLabelExpression(), null, null, null, appNodeLabel); }
private void validateLabelsRequests(ResourceRequest resourceRequest, boolean isReduce) { switch (resourceRequest.getResourceName()) { case "map": case "reduce": case NetworkTopology.DEFAULT_RACK: Assert.assertNull(resourceRequest.getNodeLabelExpression()); break; case "*": Assert.assertEquals(isReduce ? "ReduceNodes" : "MapNodes", resourceRequest.getNodeLabelExpression()); break; default: Assert.fail("Invalid resource location " + resourceRequest.getResourceName()); } }
@Test public void testNodeLabelExp() throws Exception { JobConf jobConf = new JobConf(); jobConf.set(MRJobConfig.JOB_NODE_LABEL_EXP, "GPU"); jobConf.set(MRJobConfig.AM_NODE_LABEL_EXP, "highMem"); YARNRunner yarnRunner = new YARNRunner(jobConf); ApplicationSubmissionContext appSubCtx = buildSubmitContext(yarnRunner, jobConf); assertEquals(appSubCtx.getNodeLabelExpression(), "GPU"); assertEquals(appSubCtx.getAMContainerResourceRequest() .getNodeLabelExpression(), "highMem"); }
static int getExpectedNumberOfElements(RMApp app) { int expectedNumberOfElements = 40 + 2; // 2 -> resourceRequests if (app.getApplicationSubmissionContext() .getNodeLabelExpression() != null) { expectedNumberOfElements++; } if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) { expectedNumberOfElements++; } if (AppInfo .getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()) != null) { expectedNumberOfElements++; } return expectedNumberOfElements; } }
public static ResourceRequest newResourceRequest(ResourceRequest r) { ResourceRequest request = recordFactory .newRecordInstance(ResourceRequest.class); request.setPriority(r.getPriority()); request.setResourceName(r.getResourceName()); request.setCapability(r.getCapability()); request.setNumContainers(r.getNumContainers()); request.setNodeLabelExpression(r.getNodeLabelExpression()); return request; }
private synchronized void addResourceRequest( SchedulerRequestKey schedulerKey, Map<String, ResourceRequest> requests, String resourceName, Resource capability) { ResourceRequest request = requests.get(resourceName); if (request == null) { request = org.apache.hadoop.yarn.server.utils.BuilderUtils.newResourceRequest( schedulerKey.getPriority(), resourceName, capability, 1); requests.put(resourceName, request); } else { request.setNumContainers(request.getNumContainers() + 1); } if (request.getNodeLabelExpression() == null) { request.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL); } // Note this down for next interaction with ResourceManager ask.remove(request); // clone to ensure the RM doesn't manipulate the same obj ask.add(ResourceRequest.clone(request)); if (LOG.isDebugEnabled()) { LOG.debug("addResourceRequest: applicationId=" + applicationId.getId() + " priority=" + schedulerKey.getPriority().getPriority() + " resourceName=" + resourceName + " capability=" + capability + " numContainers=" + request.getNumContainers() + " #asks=" + ask.size()); } }
public ResourceRequestInfo(ResourceRequest request) { priority = request.getPriority().getPriority(); allocationRequestId = request.getAllocationRequestId(); resourceName = request.getResourceName(); capability = new ResourceInfo(request.getCapability()); numContainers = request.getNumContainers(); relaxLocality = request.getRelaxLocality(); nodeLabelExpression = request.getNodeLabelExpression(); if (request.getExecutionTypeRequest() != null) { executionTypeRequest = new ExecutionTypeRequestInfo(request.getExecutionTypeRequest()); } }
@SuppressWarnings("rawtypes") @Override public Allocation answer(InvocationOnMock invocation) throws Throwable { ResourceRequest rr = (ResourceRequest) ((List) invocation.getArguments()[1]).get(0); // capacity shouldn't changed assertEquals(Resource.newInstance(3333, 1), rr.getCapability()); assertEquals("label-expression", rr.getNodeLabelExpression()); // priority, #container, relax-locality will be changed assertEquals(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, rr.getPriority()); assertEquals(1, rr.getNumContainers()); assertEquals(ResourceRequest.ANY, rr.getResourceName()); // just return an empty allocation List l = new ArrayList(); Set s = new HashSet(); return new Allocation(l, Resources.none(), s, s, l); } });
@SuppressWarnings("rawtypes") @Override public Allocation answer(InvocationOnMock invocation) throws Throwable { ResourceRequest rr = (ResourceRequest) ((List) invocation.getArguments()[1]).get(0); // capacity shouldn't changed assertEquals(Resource.newInstance(3333, 1), rr.getCapability()); assertEquals("label-expression", rr.getNodeLabelExpression()); // priority, #container, relax-locality will be changed assertEquals(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, rr.getPriority()); assertEquals(1, rr.getNumContainers()); assertEquals(ResourceRequest.ANY, rr.getResourceName()); // just return an empty allocation List l = new ArrayList(); Set s = new HashSet(); return new Allocation(l, Resources.none(), s, s, l); } });
/** * Clone a ResourceRequest object (shallow copy). Please keep it loaded with * all (new) fields * * @param rr the object to copy from * @return the copied object */ @Public @Evolving public static ResourceRequest clone(ResourceRequest rr) { // Please keep it loaded with all (new) fields return ResourceRequest.newBuilder().priority(rr.getPriority()) .resourceName(rr.getResourceName()).capability(rr.getCapability()) .numContainers(rr.getNumContainers()) .relaxLocality(rr.getRelaxLocality()) .nodeLabelExpression(rr.getNodeLabelExpression()) .executionTypeRequest(rr.getExecutionTypeRequest()) .allocationRequestId(rr.getAllocationRequestId()).build(); }
public void verifyResourceRequestsGeneric(ResourceRequest request, String nodeLabelExpression, int numContainers, boolean relaxLocality, int priority, String resourceName, long memory, long vCores, String executionType, boolean enforceExecutionType) { assertEquals("nodeLabelExpression doesn't match", request.getNodeLabelExpression(), nodeLabelExpression); assertEquals("numContainers doesn't match", request.getNumContainers(), numContainers); assertEquals("relaxLocality doesn't match", request.getRelaxLocality(), relaxLocality); assertEquals("priority does not match", request.getPriority().getPriority(), priority); assertEquals("resourceName does not match", request.getResourceName(), resourceName); assertEquals("memory does not match", request.getCapability().getMemorySize(), memory); assertEquals("vCores does not match", request.getCapability().getVirtualCores(), vCores); assertEquals("executionType does not match", request.getExecutionTypeRequest().getExecutionType().name(), executionType); assertEquals("enforceExecutionType does not match", request.getExecutionTypeRequest().getEnforceExecutionType(), enforceExecutionType); }