int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores); Resource capability = Resource.newInstance(jstormClientContext.amMemory, jstormClientContext.amVCores); appContext.setResource(capability); Priority pri = Priority.newInstance(jstormClientContext.amPriority); appContext.setPriority(pri);
private void requestContainer(Optional<String> preferredNode) { Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); Resource capability = Records.newRecord(Resource.class); int maxMemoryCapacity = this.maxResourceCapacity.get().getMemory(); capability.setMemory(this.requestedContainerMemoryMbs <= maxMemoryCapacity ? this.requestedContainerMemoryMbs : maxMemoryCapacity); int maxCoreCapacity = this.maxResourceCapacity.get().getVirtualCores(); capability.setVirtualCores(this.requestedContainerCores <= maxCoreCapacity ? this.requestedContainerCores : maxCoreCapacity); String[] preferredNodes = preferredNode.isPresent() ? new String[] {preferredNode.get()} : null; this.amrmClientAsync.addContainerRequest( new AMRMClient.ContainerRequest(capability, preferredNodes, null, priority)); }
if (nimbusMap.containsKey(containerId)) { Container nimbusContainer = nimbusMap.get(containerId); containerAsk = setupContainerAskForRM(nimbusContainer.getResource().getMemory(), nimbusContainer.getResource().getVirtualCores(), nimbusContainer.getPriority().getPriority(), nimbusContainer.getNodeId().getHost()); LOG.info("restart nimbus container" + ", containerId=" + containerStatus.getContainerId()); } else if (supervisorMap.containsKey(containerId)) { Container supervisorContainer = supervisorMap.get(containerId); containerAsk = setupContainerAskForRM(supervisorContainer.getResource().getMemory(), supervisorContainer.getResource().getVirtualCores(), supervisorContainer.getPriority().getPriority(), supervisorContainer.getNodeId().getHost()); LOG.info("restart supervisor container" + ", containerId=" + containerStatus.getContainerId());
public Priority createPriority(int priority) { Priority p = Records.newRecord(Priority.class); p.setPriority(priority); return p; } }
public RMApp submitApp(int masterMemory, String name, String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted, boolean keepContainers) throws Exception { Resource resource = Records.newRecord(Resource.class); resource.setMemorySize(masterMemory); return submitApp(resource, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, keepContainers, false, null, 0, null, true, Priority.newInstance(0)); }
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM */ public ContainerRequest setupContainerAskForRM(int containerMemory, int containerVirtualCores, int priority, String[] racks, String[] hosts) { Priority pri = Priority.newInstance(priority); Resource capability = Resource.newInstance(containerMemory, containerVirtualCores); ContainerRequest request = new ContainerRequest(capability, hosts, racks, pri, false); LOG.info("By Thrift Server Requested container ask: " + request.toString()); return request; }
ContainerLaunchContext amContainerLaunchContext = Records.newRecord(ContainerLaunchContext.class); amContainerLaunchContext.setLocalResources(appMasterLocalResources); amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration)); amContainerLaunchContext.setCommands(Lists.newArrayList(buildApplicationMasterCommand(resource.getMemory()))); if (UserGroupInformation.isSecurityEnabled()) { setupSecurityTokens(amContainerLaunchContext); appSubmissionContext.setResource(resource); appSubmissionContext.setQueue(this.appQueueName); appSubmissionContext.setPriority(Priority.newInstance(0)); appSubmissionContext.setAMContainerSpec(amContainerLaunchContext);
protected ResourceRequest createResourceRequest(long id, String resource, int memory, int vCores, int priority, ExecutionType execType, int containers) { ResourceRequest req = Records.newRecord(ResourceRequest.class); req.setAllocationRequestId(id); req.setResourceName(resource); req.setCapability(Resource.newInstance(memory, vCores)); req.setPriority(Priority.newInstance(priority)); req.setExecutionTypeRequest(ExecutionTypeRequest.newInstance(execType)); req.setNumContainers(containers); return req; }
public AppPriorityACLGroup(Priority maxPriority, Priority defaultPriority, AccessControlList aclList) { this.setMaxPriority(Priority.newInstance(maxPriority.getPriority())); this.setDefaultPriority( Priority.newInstance(defaultPriority.getPriority())); this.setACLList(aclList); }
private static final Priority RM_REQUEST_PRIORITY = Priority.newInstance(0);
@Override public int compare(Priority o1, Priority o2) { return o1.getPriority() - o2.getPriority(); } };
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { lastAsk = request.getAskList(); for (ResourceRequest req : lastAsk) { if (ResourceRequest.ANY.equals(req.getResourceName())) { Priority priority = req.getPriority(); if (priority.equals(RMContainerAllocator.PRIORITY_MAP)) { lastAnyAskMap = req.getNumContainers(); } else if (priority.equals(RMContainerAllocator.PRIORITY_REDUCE)){ lastAnyAskReduce = req.getNumContainers(); } } } AllocateResponse response = AllocateResponse.newInstance( request.getResponseId(), containersToComplete, containersToAllocate, Collections.<NodeReport>emptyList(), Resource.newInstance(512000, 1024), null, 10, null, Collections.<NMToken>emptyList()); // RM will always ensure that a default priority is sent to AM response.setApplicationPriority(Priority.newInstance(0)); containersToComplete.clear(); containersToAllocate.clear(); return response; }
public synchronized void addResourceRequestSpec( Priority priority, Resource capability) { Resource currentSpec = requestSpec.put(priority, capability); if (currentSpec != null) { throw new IllegalStateException("Resource spec already exists for " + "priority " + priority.getPriority() + " - " + currentSpec.getMemory()); } }
/** * Updates priority of an application with the given ID. */ private void updateApplicationPriority(String applicationId, String priority) throws YarnException, IOException { ApplicationId appId = ApplicationId.fromString(applicationId); Priority newAppPriority = Priority.newInstance(Integer.parseInt(priority)); sysout.println("Updating priority of an application " + applicationId); Priority updateApplicationPriority = client.updateApplicationPriority(appId, newAppPriority); if (newAppPriority.equals(updateApplicationPriority)) { sysout.println("Successfully updated the application " + applicationId + " with priority '" + priority + "'"); } else { sysout .println("Updated priority of an application " + applicationId + " to cluster max priority OR keeping old priority" + " as application is in final states"); } }
@Override public Priority getPriority() { Priority p = recordFactory.newRecordInstance(Priority.class); p.setPriority(1); return p; }
@Test public void testComparePriorities() { Priority high = Priority.newInstance(1); Priority low = Priority.newInstance(2); assertTrue(high.compareTo(low) > 0); }
@Override public int compare(TempAppPerPartition ta1, TempAppPerPartition ta2) { Priority p1 = Priority.newInstance(ta1.getPriority()); Priority p2 = Priority.newInstance(ta2.getPriority()); if (!p1.equals(p2)) { return p1.compareTo(p2); } return ta1.getApplicationId().compareTo(ta2.getApplicationId()); } }
@Override public int compare(ResourceRequest r1, ResourceRequest r2) { // Compare priority, host and capability int ret = r1.getPriority().compareTo(r2.getPriority()); if (ret == 0) { String h1 = r1.getResourceName(); String h2 = r2.getResourceName(); ret = h1.compareTo(h2); } if (ret == 0) { ret = r1.getCapability().compareTo(r2.getCapability()); } return ret; } }
@Override public int compareTo(TaskRequest request) { return request.priority.compareTo(this.priority); }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } TaskRequest that = (TaskRequest) o; if (priority != null ? !priority.equals(that.priority) : that.priority != null) { return false; } if (task != null ? !task.equals(that.task) : that.task != null) { return false; } return true; }