private Resource prepareContainerResource(GetNewApplicationResponse newApplicationResponse) { int memoryMbs = this.config.getInt(GobblinYarnConfigurationKeys.APP_MASTER_MEMORY_MBS_KEY); int maximumMemoryCapacity = newApplicationResponse.getMaximumResourceCapability().getMemory(); if (memoryMbs > maximumMemoryCapacity) { LOGGER.info(String.format("Specified AM memory [%d] is above the maximum memory capacity [%d] of the " + "cluster, using the maximum memory capacity instead.", memoryMbs, maximumMemoryCapacity)); memoryMbs = maximumMemoryCapacity; } int vCores = this.config.getInt(GobblinYarnConfigurationKeys.APP_MASTER_CORES_KEY); int maximumVirtualCoreCapacity = newApplicationResponse.getMaximumResourceCapability().getVirtualCores(); if (vCores > maximumVirtualCoreCapacity) { LOGGER.info(String.format("Specified AM vcores [%d] is above the maximum vcore capacity [%d] of the " + "cluster, using the maximum vcore capacity instead.", memoryMbs, maximumMemoryCapacity)); vCores = maximumVirtualCoreCapacity; } // Set up resource type requirements for ApplicationMaster return Resource.newInstance(memoryMbs, vCores); }
private void validateResources( AppSpec master ) throws ClientException { // Memory and core checks per YARN app specs. int maxMemory = appResponse.getMaximumResourceCapability().getMemory(); int maxCores = appResponse.getMaximumResourceCapability().getVirtualCores(); if (verbose) { System.out.println("Max Memory: " + maxMemory); System.out.println("Max Cores: " + maxCores); } // YARN behaves very badly if we request a container larger than the // maximum. if (master.memoryMb > maxMemory) { throw new ClientException( "YARN maximum memory is " + maxMemory + " but the application master requests " + master.memoryMb ); } if (master.vCores > maxCores) { throw new ClientException("YARN maximum vcores is " + maxCores + " but the application master requests " + master.vCores); } // Verify the limits for the Drillbit as well. if (config.getInt(DrillOnYarnConfig.DRILLBIT_MEMORY) > maxMemory) { throw new ClientException( "YARN maximum memory is " + maxMemory + " but the Drillbit requests " + config.getInt(DrillOnYarnConfig.DRILLBIT_MEMORY)); } if (config.getInt(DrillOnYarnConfig.DRILLBIT_VCORES) > maxCores) { throw new ClientException("YARN maximum vcores is " + maxCores + " but the Drillbit requests " + config.getInt(DrillOnYarnConfig.DRILLBIT_VCORES)); } }
int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores);
private void checkArguments(XLearningConfiguration conf, GetNewApplicationResponse newApplication) { int maxMem = newApplication.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capability of resources in this cluster " + maxMem); int maxVCores = newApplication.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max vcores capability of resources in this cluster " + maxVCores);
appMasterEnv.put(XLearningConstants.Environment.XLEARNING_CONTAINER_MAX_MEMORY.toString(), String.valueOf(newAppResponse.getMaximumResourceCapability().getMemory()));
final GetNewApplicationResponse appResponse = yarnApplication.getNewApplicationResponse(); Resource maxRes = appResponse.getMaximumResourceCapability();
int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores);
private void checkAmResourceRequest(GetNewApplicationResponse appResponse) { int maxMem = appResponse.getMaximumResourceCapability().getMemory(); if (amMemory > maxMem) { logger.log(Level.WARNING, "AM memory specified above max threshold of cluster. Using max value. Specified: {0}, max: {1}", new Object[]{amMemory, maxMem}); amMemory = maxMem; } int maxVcores = appResponse.getMaximumResourceCapability().getVirtualCores(); if (amVCores > maxVcores) { logger.log(Level.WARNING, "AM vcores specified above max threshold of cluster. Using max value. Specified: {0}, max: {1}", new Object[]{amVCores, maxVcores}); amVCores = maxVcores; } }
/** * Extract the desired driver memory from jobSubmissionProto. * <p> * returns maxMemory if that desired amount is more than maxMemory */ private int getMemory(final int requestedMemory) { final int maxMemory = applicationResponse.getMaximumResourceCapability().getMemory(); final int amMemory; if (requestedMemory <= maxMemory) { amMemory = requestedMemory; } else { LOG.log(Level.WARNING, "Requested {0}MB of memory for the driver. " + "The max on this YARN installation is {1}. " + "Using {1} as the memory for the driver.", new Object[]{requestedMemory, maxMemory}); amMemory = maxMemory; } return amMemory; }
private Resource prepareContainerResource(GetNewApplicationResponse newApplicationResponse) { int memoryMbs = this.config.getInt(GobblinYarnConfigurationKeys.APP_MASTER_MEMORY_MBS_KEY); int maximumMemoryCapacity = newApplicationResponse.getMaximumResourceCapability().getMemory(); if (memoryMbs > maximumMemoryCapacity) { LOGGER.info(String.format("Specified AM memory [%d] is above the maximum memory capacity [%d] of the " + "cluster, using the maximum memory capacity instead.", memoryMbs, maximumMemoryCapacity)); memoryMbs = maximumMemoryCapacity; } int vCores = this.config.getInt(GobblinYarnConfigurationKeys.APP_MASTER_CORES_KEY); int maximumVirtualCoreCapacity = newApplicationResponse.getMaximumResourceCapability().getVirtualCores(); if (vCores > maximumVirtualCoreCapacity) { LOGGER.info(String.format("Specified AM vcores [%d] is above the maximum vcore capacity [%d] of the " + "cluster, using the maximum vcore capacity instead.", memoryMbs, maximumMemoryCapacity)); vCores = maximumVirtualCoreCapacity; } // Set up resource type requirements for ApplicationMaster return Resource.newInstance(memoryMbs, vCores); }
/** * Adjust the user-supplied <code>-yh</code> and <code>-w</code> settings if they are too small or large for the * current cluster, and re-record the new settings in the Configuration for export. * * @param gnar * the GetNewAppResponse from the YARN ResourceManager. */ private void checkAndAdjustPerTaskHeapSize(final GetNewApplicationResponse gnar) { final int maxCapacity = gnar.getMaximumResourceCapability().getMemory(); // make sure heap size is OK for this cluster's available containers int guaguaYarnMem = getConf().getInt(GuaguaYarnConstants.GUAGUA_CHILD_MEMORY, GuaguaYarnConstants.GUAGUA_CHILD_DEFAULT_MEMORY); if(guaguaYarnMem > maxCapacity) { LOG.warn("Guagua's request of heap MB per-task is more than the minimum; downgrading guagua to {} MB.", maxCapacity); guaguaYarnMem = maxCapacity; } getConf().setInt(GuaguaYarnConstants.GUAGUA_CHILD_MEMORY, guaguaYarnMem); }
/** * Adjust the user-supplied <code>-yh</code> and <code>-w</code> settings if they are too small or large for the * current cluster, and re-record the new settings in the Configuration for export. * * @param gnar * the GetNewAppResponse from the YARN ResourceManager. */ private void checkAndAdjustPerTaskHeapSize(final GetNewApplicationResponse gnar) { final int maxCapacity = gnar.getMaximumResourceCapability().getMemory(); // make sure heap size is OK for this cluster's available containers int guaguaYarnMem = getConf().getInt(GuaguaYarnConstants.GUAGUA_CHILD_MEMORY, GuaguaYarnConstants.GUAGUA_CHILD_DEFAULT_MEMORY); if(guaguaYarnMem > maxCapacity) { LOG.warn("Guagua's request of heap MB per-task is more than the minimum; downgrading guagua to {} MB.", maxCapacity); guaguaYarnMem = maxCapacity; } getConf().setInt(GuaguaYarnConstants.GUAGUA_CHILD_MEMORY, guaguaYarnMem); }
private Resource adjustMemory(GetNewApplicationResponse response, Resource capability) { int maxMemory = response.getMaximumResourceCapability().getMemory(); int updatedMemory = capability.getMemory(); if (updatedMemory > maxMemory) { capability.setMemory(maxMemory); } return capability; }
private Resource adjustMemory(GetNewApplicationResponse response, Resource capability) { int maxMemory = response.getMaximumResourceCapability().getMemory(); int updatedMemory = capability.getMemory(); if (updatedMemory > maxMemory) { capability.setMemory(maxMemory); } return capability; }
private Resource adjustMemory(GetNewApplicationResponse response, Resource capability) { int minMemory = response.getMinimumResourceCapability().getMemory(); int updatedMemory = Math.min(capability.getMemory(), response.getMaximumResourceCapability().getMemory()); updatedMemory = (int) Math.ceil(((double) updatedMemory / minMemory)) * minMemory; if (updatedMemory != capability.getMemory()) { capability.setMemory(updatedMemory); } return capability; }
/** * Function that actually creates the ApplicationId by calling the * ClientRMService * * @return returns structure containing the app-id and maximum resource * capabilities */ private NewApplication createNewApplication() { GetNewApplicationRequest req = recordFactory.newRecordInstance(GetNewApplicationRequest.class); GetNewApplicationResponse resp; try { resp = rm.getClientRMService().getNewApplication(req); } catch (YarnException e) { String msg = "Unable to create new app from RM web service"; LOG.error(msg, e); throw new YarnRuntimeException(msg, e); } NewApplication appId = new NewApplication(resp.getApplicationId().toString(), new ResourceInfo(resp.getMaximumResourceCapability())); return appId; }
/** * Function that actually creates the ApplicationId by calling the * ClientRMService * * @return returns structure containing the app-id and maximum resource * capabilities */ private NewApplication createNewApplication() { GetNewApplicationRequest req = recordFactory.newRecordInstance(GetNewApplicationRequest.class); GetNewApplicationResponse resp; try { resp = rm.getClientRMService().getNewApplication(req); } catch (YarnException e) { String msg = "Unable to create new app from RM web service"; LOG.error(msg, e); throw new YarnRuntimeException(msg, e); } NewApplication appId = new NewApplication(resp.getApplicationId().toString(), new ResourceInfo(resp.getMaximumResourceCapability())); return appId; }
/** * Function that actually creates the ApplicationId by calling the * ClientRMService * * @return returns structure containing the app-id and maximum resource * capabilities */ private NewApplication createNewApplication() { GetNewApplicationRequest req = recordFactory.newRecordInstance(GetNewApplicationRequest.class); GetNewApplicationResponse resp; try { resp = rm.getClientRMService().getNewApplication(req); } catch (YarnException e) { String msg = "Unable to create new app from RM web service"; LOG.error(msg, e); throw new YarnRuntimeException(msg, e); } NewApplication appId = new NewApplication(resp.getApplicationId().toString(), new ResourceInfo(resp.getMaximumResourceCapability())); return appId; }
@Test public void testGetNewAppId() throws Exception { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); MockRM rm = new MockRM(conf); rm.start(); GetNewApplicationResponse resp = rm.getNewAppId(); assert (resp.getApplicationId().getId() != 0); assert (resp.getMaximumResourceCapability().getMemorySize() > 0); rm.stop(); }
@Test public void testGetNewAppId() throws Exception { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); MockRM rm = new MockRM(conf); rm.start(); GetNewApplicationResponse resp = rm.getNewAppId(); assert (resp.getApplicationId().getId() != 0); assert (resp.getMaximumResourceCapability().getMemory() > 0); rm.stop(); }