applicationId = newAppResponse.getApplicationId(); LOG.info("Got new Application: " + applicationId.toString()); appMasterEnv.put(XLearningConstants.Environment.XLEARNING_CONTAINER_MAX_MEMORY.toString(), String.valueOf(newAppResponse.getMaximumResourceCapability().getMemory()));
/** * Kills YARN application and stops YARN client. * * <p>Use this method to kill the App before it has been properly deployed */ private void failSessionDuringDeployment(YarnClient yarnClient, YarnClientApplication yarnApplication) { LOG.info("Killing YARN application"); try { yarnClient.killApplication(yarnApplication.getNewApplicationResponse().getApplicationId()); } catch (Exception e) { // we only log a debug message here because the "killApplication" call is a best-effort // call (we don't know if the application has been deployed when the error occured). LOG.debug("Error while killing YARN application", e); } yarnClient.stop(); }
private Resource prepareContainerResource(GetNewApplicationResponse newApplicationResponse) { int memoryMbs = this.config.getInt(GobblinYarnConfigurationKeys.APP_MASTER_MEMORY_MBS_KEY); int maximumMemoryCapacity = newApplicationResponse.getMaximumResourceCapability().getMemory(); if (memoryMbs > maximumMemoryCapacity) { LOGGER.info(String.format("Specified AM memory [%d] is above the maximum memory capacity [%d] of the " + "cluster, using the maximum memory capacity instead.", memoryMbs, maximumMemoryCapacity)); memoryMbs = maximumMemoryCapacity; } int vCores = this.config.getInt(GobblinYarnConfigurationKeys.APP_MASTER_CORES_KEY); int maximumVirtualCoreCapacity = newApplicationResponse.getMaximumResourceCapability().getVirtualCores(); if (vCores > maximumVirtualCoreCapacity) { LOGGER.info(String.format("Specified AM vcores [%d] is above the maximum vcore capacity [%d] of the " + "cluster, using the maximum vcore capacity instead.", memoryMbs, maximumMemoryCapacity)); vCores = maximumVirtualCoreCapacity; } // Set up resource type requirements for ApplicationMaster return Resource.newInstance(memoryMbs, vCores); }
@Override public GetNewApplicationResponse getNewApplication( GetNewApplicationRequest request) throws YarnException { GetNewApplicationResponse response = recordFactory .newRecordInstance(GetNewApplicationResponse.class); response.setApplicationId(getNewApplicationId()); // Pick up min/max resource from scheduler... response.setMaximumResourceCapability(scheduler .getMaximumResourceCapability()); return response; }
private Resource adjustMemory(GetNewApplicationResponse response, Resource capability) { int minMemory = response.getMinimumResourceCapability().getMemory(); int updatedMemory = Math.min(capability.getMemory(), response.getMaximumResourceCapability().getMemory()); updatedMemory = (int) Math.ceil(((double) updatedMemory / minMemory)) * minMemory; if (updatedMemory != capability.getMemory()) { capability.setMemory(updatedMemory); } return capability; }
@Override public YarnClientApplication createApplication() throws YarnException, IOException { ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class); ApplicationId appId = ApplicationId.newInstance(clusterTimeStamp, appIdNumber++); context.setApplicationId(appId); GetNewApplicationResponse response = Records.newRecord(GetNewApplicationResponse.class); response.setApplicationId(appId); return new YarnClientApplication(response, context); }
private void createApp() throws ClientException { try { appResponse = client.createAppMaster(); } catch (YarnClientException e) { throw new ClientException("Failed to allocate Drill application master", e); } appId = appResponse.getApplicationId(); System.out.println("Application ID: " + appId.toString()); }
private void validateResources( AppSpec master ) throws ClientException { // Memory and core checks per YARN app specs. int maxMemory = appResponse.getMaximumResourceCapability().getMemory(); int maxCores = appResponse.getMaximumResourceCapability().getVirtualCores(); if (verbose) { System.out.println("Max Memory: " + maxMemory); System.out.println("Max Cores: " + maxCores); } // YARN behaves very badly if we request a container larger than the // maximum. if (master.memoryMb > maxMemory) { throw new ClientException( "YARN maximum memory is " + maxMemory + " but the application master requests " + master.memoryMb ); } if (master.vCores > maxCores) { throw new ClientException("YARN maximum vcores is " + maxCores + " but the application master requests " + master.vCores); } // Verify the limits for the Drillbit as well. if (config.getInt(DrillOnYarnConfig.DRILLBIT_MEMORY) > maxMemory) { throw new ClientException( "YARN maximum memory is " + maxMemory + " but the Drillbit requests " + config.getInt(DrillOnYarnConfig.DRILLBIT_MEMORY)); } if (config.getInt(DrillOnYarnConfig.DRILLBIT_VCORES) > maxCores) { throw new ClientException("YARN maximum vcores is " + maxCores + " but the Drillbit requests " + config.getInt(DrillOnYarnConfig.DRILLBIT_VCORES)); } }
@Private @Unstable public static GetNewApplicationResponse newInstance( ApplicationId applicationId, Resource minCapability, Resource maxCapability) { GetNewApplicationResponse response = Records.newRecord(GetNewApplicationResponse.class); response.setApplicationId(applicationId); response.setMaximumResourceCapability(maxCapability); return response; }
newAppResponse.setApplicationId(appId); when(clientRMProtocol.getNewApplication(any(GetNewApplicationRequest.class))). thenReturn(newAppResponse);
/** * Function that actually creates the ApplicationId by calling the * ClientRMService * * @return returns structure containing the app-id and maximum resource * capabilities */ private NewApplication createNewApplication() { GetNewApplicationRequest req = recordFactory.newRecordInstance(GetNewApplicationRequest.class); GetNewApplicationResponse resp; try { resp = rm.getClientRMService().getNewApplication(req); } catch (YarnException e) { String msg = "Unable to create new app from RM web service"; LOG.error(msg, e); throw new YarnRuntimeException(msg, e); } NewApplication appId = new NewApplication(resp.getApplicationId().toString(), new ResourceInfo(resp.getMaximumResourceCapability())); return appId; }
public GetNewApplicationResponse createAppMaster() throws YarnClientException { // Create application via yarnClient // Response is a new application ID along with cluster capacity info try { app = yarnClient.createApplication(); } catch (YarnException | IOException e) { throw new YarnClientException("Create application failed", e); } GetNewApplicationResponse response = app.getNewApplicationResponse(); appId = response.getApplicationId(); return response; }
private void checkArguments(XLearningConfiguration conf, GetNewApplicationResponse newApplication) { int maxMem = newApplication.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capability of resources in this cluster " + maxMem); int maxVCores = newApplication.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max vcores capability of resources in this cluster " + maxVCores);
@Override public GetNewApplicationResponse getNewApplication( GetNewApplicationRequest request) throws YarnException { GetNewApplicationResponse response = recordFactory .newRecordInstance(GetNewApplicationResponse.class); response.setApplicationId(getNewApplicationId()); // Pick up min/max resource from scheduler... response.setMaximumResourceCapability(scheduler .getMaximumResourceCapability()); return response; }
/** * Function that actually creates the ApplicationId by calling the * ClientRMService * * @return returns structure containing the app-id and maximum resource * capabilities */ private NewApplication createNewApplication() { GetNewApplicationRequest req = recordFactory.newRecordInstance(GetNewApplicationRequest.class); GetNewApplicationResponse resp; try { resp = rm.getClientRMService().getNewApplication(req); } catch (YarnException e) { String msg = "Unable to create new app from RM web service"; LOG.error(msg, e); throw new YarnRuntimeException(msg, e); } NewApplication appId = new NewApplication(resp.getApplicationId().toString(), new ResourceInfo(resp.getMaximumResourceCapability())); return appId; }
public ApplicationId getApplicationId() throws YarnRemoteException { if (clientResourceManager == null) throw new IllegalStateException( "Cannot get an application ID befire connecting to resource manager!"); GetNewApplicationRequest appReq = Records.newRecord(GetNewApplicationRequest.class); GetNewApplicationResponse appRes = clientResourceManager.getNewApplication(appReq); LOG.info("Got a new application with id=" + appRes.getApplicationId()); return appRes.getApplicationId(); }
int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores);
@Private @Unstable public static GetNewApplicationResponse newInstance( ApplicationId applicationId, Resource minCapability, Resource maxCapability) { GetNewApplicationResponse response = Records.newRecord(GetNewApplicationResponse.class); response.setApplicationId(applicationId); response.setMaximumResourceCapability(maxCapability); return response; }
/** * Function that actually creates the ApplicationId by calling the * ClientRMService * * @return returns structure containing the app-id and maximum resource * capabilities */ private NewApplication createNewApplication() { GetNewApplicationRequest req = recordFactory.newRecordInstance(GetNewApplicationRequest.class); GetNewApplicationResponse resp; try { resp = rm.getClientRMService().getNewApplication(req); } catch (YarnException e) { String msg = "Unable to create new app from RM web service"; LOG.error(msg, e); throw new YarnRuntimeException(msg, e); } NewApplication appId = new NewApplication(resp.getApplicationId().toString(), new ResourceInfo(resp.getMaximumResourceCapability())); return appId; }
/** * Get a new application from the ASM * * @return New Application * @throws IOException * , YarnException */ private GetNewApplicationResponse getApplication() throws IOException, YarnException { GetNewApplicationRequest request = Records .newRecord(GetNewApplicationRequest.class); GetNewApplicationResponse response = applicationsManager .getNewApplication(request); LOG.info("Got new application id=" + response.getApplicationId()); return response; }