/** * Propagate an option from the cluster specification option map * to the site XML map, using the site key for the name * @param global global config spec * @param optionKey key in the option map * @param sitexml map for XML file to build up * @param siteKey key to assign the value to in the site XML * @throws BadConfigException if the option is missing from the cluster spec */ public void propagateOption(MapOperations global, String optionKey, Map<String, String> sitexml, String siteKey) throws BadConfigException { sitexml.put(siteKey, global.getMandatoryOption(optionKey)); }
@Override String getAlias(MapOperations compOps) { return compOps.getOption(SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_KEY, SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT); } }
/** * Return the label expression and if not set null * @param map map to look up * @return extracted label or null */ public String extractLabelExpression(Map<String, String> map) { if (map != null) { MapOperations options = new MapOperations("", map); return options.getOption(ResourceKeys.YARN_LABEL_EXPRESSION, null); } return null; }
/** * Get a mandatory integer option; use {@link Integer#decode(String)} so as to take hex * oct and bin values too. * * @param option option name * @return parsed value * @throws NumberFormatException if the option could not be parsed. * @throws BadConfigException if the option could not be found */ public int getMandatoryOptionInt(String option) throws BadConfigException { getMandatoryOption(option); return getOptionInt(option, 0); }
MapOperations appInformation = new MapOperations(); Map<String, String> sliderAppConfKeys = amConfiguration.prefixedWith("slider."); for (Map.Entry<String, String> entry : sliderAppConfKeys.entrySet()) { String k = entry.getKey(); String providerType = globalInternalOptions.getMandatoryOption( InternalKeys.INTERNAL_PROVIDER_NAME); log.info("Cluster provider type is {}", providerType); appInformation.put(StatusKeys.INFO_AM_CONTAINER_ID, appMasterContainerID.toString()); appInformation.put(StatusKeys.INFO_AM_APP_ID, appid.toString()); appInformation.put(StatusKeys.INFO_AM_ATTEMPT_ID, appAttemptID.toString()); appMasterTrackingUrl = null; log.info("AM Server is listening at {}:{}", appMasterHostname, appMasterRpcPort); appInformation.put(StatusKeys.INFO_AM_HOSTNAME, appMasterHostname); appInformation.set(StatusKeys.INFO_AM_RPC_PORT, appMasterRpcPort); if (component.getOptionBool( AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false)) { uploadServerCertForLocalization(clustername, fs); appInformation.put(StatusKeys.INFO_AM_WEB_URL, appMasterTrackingUrl + "/"); appInformation.set(StatusKeys.INFO_AM_WEB_PORT, webAppPort); appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(maxCores)); appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(maxMemory));
/** * Build a dynamic provider role * @param name name of role * @param group group of role * @return a new provider role * @throws BadConfigException bad configuration */ public ProviderRole createDynamicProviderRole(String name, String group, MapOperations component) throws BadConfigException { String priOpt = component.getMandatoryOption(COMPONENT_PRIORITY); int priority = SliderUtils.parseAndValidate( "value of " + name + " " + COMPONENT_PRIORITY, priOpt, 0, 1, -1); String placementOpt = component.getOption(COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy.DEFAULT)); int placement = SliderUtils.parseAndValidate( "value of " + name + " " + COMPONENT_PLACEMENT_POLICY, placementOpt, 0, 0, -1); int placementTimeout = component.getOptionInt(PLACEMENT_ESCALATE_DELAY, DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS); ProviderRole newRole = new ProviderRole(name, group, priority, placement, getNodeFailureThresholdForRole(group), placementTimeout, component.getOption(YARN_LABEL_EXPRESSION, DEF_YARN_LABEL_EXPRESSION)); log.info("New {} ", newRole); return newRole; }
SliderUtils.mergeMaps(subComponentOps, ops.getComponent(externalComponent).options); subComponentOps.remove(COMPONENT_TYPE_KEY); String existingPrefix = subComponentOps.get(ROLE_PREFIX); if (existingPrefix != null) { if (!subComponent.startsWith(existingPrefix)) { subComponentOps.set(ROLE_PREFIX, prefix); subComponentOps.put(ResourceKeys.COMPONENT_PRIORITY, Integer.toString(priority)); priorities.add(priority);
/** * return the HDFS path where the application package has been uploaded * manually or by using slider client (install package command) * * @param conf configuration * @param roleGroup name of component * @return */ public static String getApplicationDefinitionPath(ConfTreeOperations conf, String roleGroup) throws BadConfigException { String appDefPath = conf.getGlobalOptions().getMandatoryOption( AgentKeys.APP_DEF); if (roleGroup != null) { MapOperations component = conf.getComponent(roleGroup); if (component != null) { appDefPath = component.getOption(AgentKeys.APP_DEF, appDefPath); } } return appDefPath; }
appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath()); appConf.getGlobalOptions().remove(AgentKeys.APP_DEF); Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip")); log.info(appDefinitions.get(0).toString()); Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith("default")); appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath()); appConf.getGlobalOptions().remove(AgentKeys.APP_DEF); adp.processSuppliedDefinitions(clustername, buildInfo, appConf); appDefinitions = adp.getAppDefinitions(); Assert.assertTrue(appDefinitions.size() == 1); Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip")); log.info(appDefinitions.get(0).toString()); Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith(tempDir.toString())); buildInfo.appDef = null; buildInfo.appMetaInfo = null; appConf.getGlobalOptions().remove(AgentKeys.APP_DEF); appConf.getGlobalOptions().remove(AgentKeys.APP_DEF); Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip")); Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon1").contains( "addons/addon1/addon_addon1.zip")); Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon2").contains( "addons/addon2/addon_addon2.zip"));
@Before public void setup() throws Exception { certMan = new CertificateManager(); MapOperations compOperations = new MapOperations(); secDir = new File(workDir.getRoot(), SliderKeys.SECURITY_DIR); File keystoreFile = new File(secDir, SliderKeys.KEYSTORE_FILE_NAME); compOperations.put(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION, keystoreFile.getAbsolutePath()); certMan.initialize(compOperations, "cahost", null, null); }
/** * Extract any resource requirements from this component's settings. * All fields that are set will override the existing values -if * unset that resource field will be left unchanged. * * Important: the configuration must already be fully resolved * in order to pick up global options. * @param resource resource to configure * @param map map of options */ public void extractResourceRequirements(Resource resource, Map<String, String> map) { if (map != null) { MapOperations options = new MapOperations("", map); resource.setMemory(options.getOptionInt(ResourceKeys.YARN_MEMORY, resource.getMemory())); resource.setVirtualCores(options.getOptionInt(ResourceKeys.YARN_CORES, resource.getVirtualCores())); } }
/** * Set a global option, converting it to a string as needed * @param key key * @param value non null value */ public void set(String key, Object value) { globalOptions.put(key, value.toString()); } /**
public ConfTreeOperations(ConfTree confTree) { assert confTree != null : "null tree"; assert confTree.components != null : "null tree components"; this.confTree = confTree; globalOptions = new MapOperations("global", confTree.global); }
/** * Update the AM resource with any local needs * @param capability capability to update */ public void prepareAMResourceRequirements(MapOperations sliderAM, Resource capability) { capability.setMemory(sliderAM.getOptionInt( ResourceKeys.YARN_MEMORY, capability.getMemory())); capability.setVirtualCores( sliderAM.getOptionInt(ResourceKeys.YARN_CORES, capability.getVirtualCores())); }
/** * Important: the configuration must already be fully resolved * in order to pick up global options * Copy env vars into the launch context. */ public boolean copyEnvVars(MapOperations options) { if (options == null) { return false; } for (Map.Entry<String, String> entry : options.entrySet()) { String key = entry.getKey(); if (key.startsWith(RoleKeys.ENV_PREFIX)) { key = key.substring(RoleKeys.ENV_PREFIX.length()); env.put(key, entry.getValue()); } } return true; }
addService(agentService); appInformation.put(StatusKeys.INFO_AM_AGENT_OPS_URL, agentOpsUrl + "/"); appInformation.put(StatusKeys.INFO_AM_AGENT_STATUS_URL, agentStatusUrl + "/"); appInformation.set(StatusKeys.INFO_AM_AGENT_STATUS_PORT, agentWebApp.getPort()); appInformation.set(StatusKeys.INFO_AM_AGENT_OPS_PORT, agentWebApp.getSecuredPort());
if (SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) { throw new BadConfigException( "%s cannot not be set if %s is specified in the cmd line ", SliderKeys.DEFAULT_APP_PKG)); Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG); appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath); log.info("Setting app package to {}.", appDefPath); if (SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) { throw new BadConfigException("application.def must not be set if --appdef is provided."); appDefinitions.add(new AppDefinition(appDirPath, buildInfo.appDef, SliderKeys.DEFAULT_APP_PKG)); Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG); appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath); log.info("Setting app package to {}.", appDefPath); if (SliderUtils.isUnset(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) { throw new BadConfigException("addon package can only be specified if main app package is specified."); Path addonPkgPath = new Path(addonPath, addonPkgName); log.info("Setting addon package {} to {}.", addOnKey, addonPkgPath); appConf.getGlobalOptions().set(addOnKey, addonPkgPath); addons.add(addOnKey); String existingList = appConf.getGlobalOptions().get(AgentKeys.ADDONS); if (SliderUtils.isUnset(existingList)) { existingList = "";
public String[] dumpEnvToString() { List<String> nodeEnv = new ArrayList<>(); for (Map.Entry<String, String> entry : env.entrySet()) { String envElt = String.format("%s=\"%s\"", entry.getKey(), entry.getValue()); log.debug(envElt); nodeEnv.add(envElt); } String[] envDescription = nodeEnv.toArray(new String[nodeEnv.size()]); return envDescription; }