return new ContainerRequest(capability, nodeArr, rackArr, priorityRec, relaxLocality, nodeExpr);
workerCapability.setMemory(workerMemory); workerCapability.setVirtualCores(workerVCores); workerContainerRequest = new ContainerRequest(workerCapability, hostLocals, null, priority); LOG.info("Create worker container request: " + workerContainerRequest.toString()); psCapability.setMemory(psMemory); psCapability.setVirtualCores(psVCores); psContainerRequest = new ContainerRequest(psCapability, hostLocals, null, priority); LOG.info("Create ps container request: " + psContainerRequest.toString());
ContainerRequest nnContainerRequest = setupContainerAskForRM(amOptions.getNameNodeMemoryMB(), amOptions.getNameNodeVirtualCores(), 0, amOptions.getNameNodeNodeLabelExpression()); LOG.info("Requested NameNode ask: " + nnContainerRequest.toString()); amRMClient.addContainerRequest(nnContainerRequest); amOptions.getDataNodeVirtualCores(), 1, amOptions.getDataNodeNodeLabelExpression()); amRMClient.addContainerRequest(datanodeAsk); LOG.debug("Requested datanode ask: " + datanodeAsk.toString());
issuedRequest = new AMRMClient.ContainerRequest(resource, hosts, null,
issuedRequest = new AMRMClient.ContainerRequest(resource, hosts, null,
this.containers.add(container); final AMRMClient.ContainerRequest requestToBeSubmitted = this.outstandingContainerRequests.peek(); final int currentRequestMemory = requestToBeSubmitted.getCapability().getMemory();
ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority); System.out.println("Making res-req " + i); rmClient.addContainerRequest(containerAsk);
"Resource request can not be null."); Set<String> dedupedRacks = new HashSet<String>(); if (req.getRacks() != null) { dedupedRacks.addAll(req.getRacks()); if(req.getRacks().size() != dedupedRacks.size()) { Joiner joiner = Joiner.on(','); LOG.warn("ContainerRequest has duplicate racks: " + joiner.join(req.getRacks())); Set<String> inferredRacks = resolveRacks(req.getNodes()); inferredRacks.removeAll(dedupedRacks); Resource resource = checkAndGetResourceProfile(req.getResourceProfile(), req.getCapability()); checkLocalityRelaxationConflict(req.getAllocationRequestId(), req.getPriority(), ANY_LIST, req.getRelaxLocality()); checkLocalityRelaxationConflict(req.getAllocationRequestId(), req.getPriority(), dedupedRacks, true); checkLocalityRelaxationConflict(req.getAllocationRequestId(), req.getPriority(), inferredRacks, req.getRelaxLocality()); if (req.getNodes() != null) { HashSet<String> dedupedNodes = new HashSet<String>(req.getNodes()); if(dedupedNodes.size() != req.getNodes().size()) { Joiner joiner = Joiner.on(','); LOG.warn("ContainerRequest has duplicate nodes: " + joiner.join(req.getNodes()));
"Resource request can not be null."); Set<String> dedupedRacks = new HashSet<String>(); if (req.getRacks() != null) { dedupedRacks.addAll(req.getRacks()); if(req.getRacks().size() != dedupedRacks.size()) { Joiner joiner = Joiner.on(','); LOG.warn("ContainerRequest has duplicate racks: " + joiner.join(req.getRacks())); Set<String> inferredRacks = resolveRacks(req.getNodes()); inferredRacks.removeAll(dedupedRacks); checkLocalityRelaxationConflict(req.getPriority(), ANY_LIST, req.getRelaxLocality()); checkLocalityRelaxationConflict(req.getPriority(), dedupedRacks, true); checkLocalityRelaxationConflict(req.getPriority(), inferredRacks, req.getRelaxLocality()); if (req.getNodes() != null) { HashSet<String> dedupedNodes = new HashSet<String>(req.getNodes()); if(dedupedNodes.size() != req.getNodes().size()) { Joiner joiner = Joiner.on(','); LOG.warn("ContainerRequest has duplicate nodes: " + joiner.join(req.getNodes())); addResourceRequest(req.getPriority(), node, req.getCapability(), req, true, req.getNodeLabelExpression());
"Resource request can not be null."); Set<String> dedupedRacks = new HashSet<String>(); if (req.getRacks() != null) { dedupedRacks.addAll(req.getRacks()); if(req.getRacks().size() != dedupedRacks.size()) { Joiner joiner = Joiner.on(','); LOG.warn("ContainerRequest has duplicate racks: " + joiner.join(req.getRacks())); Set<String> inferredRacks = resolveRacks(req.getNodes()); inferredRacks.removeAll(dedupedRacks); checkLocalityRelaxationConflict(req.getPriority(), ANY_LIST, req.getRelaxLocality()); checkLocalityRelaxationConflict(req.getPriority(), dedupedRacks, true); checkLocalityRelaxationConflict(req.getPriority(), inferredRacks, req.getRelaxLocality()); if (req.getNodes() != null) { HashSet<String> dedupedNodes = new HashSet<String>(req.getNodes()); if(dedupedNodes.size() != req.getNodes().size()) { Joiner joiner = Joiner.on(','); LOG.warn("ContainerRequest has duplicate nodes: " + joiner.join(req.getNodes())); addResourceRequest(req.getPriority(), node, req.getCapability(), req, true, req.getNodeLabelExpression());
"Resource request can not be null."); Set<String> dedupedRacks = new HashSet<String>(); if (req.getRacks() != null) { dedupedRacks.addAll(req.getRacks()); if(req.getRacks().size() != dedupedRacks.size()) { Joiner joiner = Joiner.on(','); LOG.warn("ContainerRequest has duplicate racks: " + joiner.join(req.getRacks())); Set<String> inferredRacks = resolveRacks(req.getNodes()); inferredRacks.removeAll(dedupedRacks); checkLocalityRelaxationConflict(req.getPriority(), ANY_LIST, req.getRelaxLocality()); checkLocalityRelaxationConflict(req.getPriority(), dedupedRacks, true); checkLocalityRelaxationConflict(req.getPriority(), inferredRacks, req.getRelaxLocality()); if (req.getNodes() != null) { HashSet<String> dedupedNodes = new HashSet<String>(req.getNodes()); if(dedupedNodes.size() != req.getNodes().size()) { Joiner joiner = Joiner.on(','); LOG.warn("ContainerRequest has duplicate nodes: " + joiner.join(req.getNodes())); addResourceRequest(req.getPriority(), node, req.getCapability(), req, true, req.getNodeLabelExpression());
private ContainerRequest containerRequest = new ContainerRequest();
List<String> blacklistNodes = resourceRequestor.getNodesExceptHost(requests.get(0).getNodes()); amRmClient.updateBlacklist(blacklistNodes, requests.get(0).getNodes()); blacklistedNodesForHostSpecificRequests = blacklistNodes; LOG.info("Sending {} request(s) after blacklisting all nodes other than {}", requests.size(), requests.get(0).getNodes()); ContainerRequest newCr = new ContainerRequest(cr.getCapability(), null, null, cr.getPriority()); MutablePair<Integer, ContainerRequest> pair = new MutablePair<>(loopCounter, newCr); requestedResources.put(csr, pair);
issuedRequest = new AMRMClient.ContainerRequest(capability, null, null, Priority.newInstance(ANY_HOST_PRIORITY), true, containerLabel); issuedRequest = new AMRMClient.ContainerRequest(capability, new String[]{preferredHost}, null, Priority.newInstance(PREFERRED_HOST_PRIORITY), false, containerLabel);
issuedRequest = new AMRMClient.ContainerRequest(capability, null, null, Priority.newInstance(ANY_HOST_PRIORITY), true, containerLabel); issuedRequest = new AMRMClient.ContainerRequest(capability, new String[]{preferredHost}, null, Priority.newInstance(PREFERRED_HOST_PRIORITY), false, containerLabel);
List<String> issuedRequestNodes = issuedRequest.getNodes(); if (SliderUtils.isUnset(label) && issuedRequestNodes != null) { nodes = issuedRequestNodes.toArray(new String[issuedRequestNodes.size()]); issuedRequest = new AMRMClient.ContainerRequest(issuedRequest.getCapability(), nodes, null,
ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority); System.out.println("[AM] Making res-req " + i); rmClient.addContainerRequest(containerAsk);
List<String> issuedRequestNodes = issuedRequest.getNodes(); if (SliderUtils.isUnset(label) && issuedRequestNodes != null) { nodes = issuedRequestNodes.toArray(new String[issuedRequestNodes.size()]); issuedRequest = new AMRMClient.ContainerRequest(issuedRequest.getCapability(), nodes, null,
@Override protected void requestNewWorkers(int numWorkers) { final long mem = taskManagerParameters.taskManagerTotalMemoryMB(); final int containerMemorySizeMB; if (mem <= Integer.MAX_VALUE) { containerMemorySizeMB = (int) mem; } else { containerMemorySizeMB = Integer.MAX_VALUE; LOG.error("Decreasing container size from {} MB to {} MB (integer value overflow)", mem, containerMemorySizeMB); } for (int i = 0; i < numWorkers; i++) { numPendingContainerRequests++; LOG.info("Requesting new TaskManager container with {} megabytes memory. Pending requests: {}", containerMemorySizeMB, numPendingContainerRequests); // Priority for worker containers - priorities are intra-application Priority priority = Priority.newInstance(0); // Resource requirements for worker containers int taskManagerSlots = taskManagerParameters.numSlots(); int vcores = config.getInteger(YarnConfigOptions.VCORES, Math.max(taskManagerSlots, 1)); Resource capability = Resource.newInstance(containerMemorySizeMB, vcores); resourceManagerClient.addContainerRequest( new AMRMClient.ContainerRequest(capability, null, null, priority)); } // make sure we transmit the request fast and receive fast news of granted allocations resourceManagerClient.setHeartbeatInterval(FAST_YARN_HEARTBEAT_INTERVAL_MS); }
/** * Inner Validation logic for container request * @param containerRequest request * @param priority raw priority of role * @param requestDetails details for error messages */ @VisibleForTesting public static void validateContainerRequest(AMRMClient.ContainerRequest containerRequest, int priority, String requestDetails) { String exp = containerRequest.getNodeLabelExpression(); boolean hasRacks = containerRequest.getRacks() != null && (!containerRequest.getRacks().isEmpty()); boolean hasNodes = containerRequest.getNodes() != null && (!containerRequest.getNodes().isEmpty()); boolean hasLabel = SliderUtils.isSet(exp); // Don't support specifying >= 2 node labels in a node label expression now if (hasLabel && (exp.contains("&&") || exp.contains("||"))) { throw new InvalidContainerRequestException( "Cannot specify more than two node labels" + " in a single node label expression: " + requestDetails); } // Don't allow specify node label against ANY request listing hosts or racks if (hasLabel && ( hasRacks || hasNodes)) { throw new InvalidContainerRequestException( "Cannot specify node label with rack or node: " + requestDetails); } }