/** * Propagate an option from the cluster specification option map * to the site XML map, using the site key for the name * @param global global config spec * @param optionKey key in the option map * @param sitexml map for XML file to build up * @param siteKey key to assign the value to in the site XML * @throws BadConfigException if the option is missing from the cluster spec */ public void propagateOption(MapOperations global, String optionKey, Map<String, String> sitexml, String siteKey) throws BadConfigException { sitexml.put(siteKey, global.getMandatoryOption(optionKey)); }
/** * Propagate an option from the cluster specification option map * to the site XML map, using the site key for the name * @param global global config spec * @param optionKey key in the option map * @param sitexml map for XML file to build up * @param siteKey key to assign the value to in the site XML * @throws BadConfigException if the option is missing from the cluster spec */ public void propagateOption(MapOperations global, String optionKey, Map<String, String> sitexml, String siteKey) throws BadConfigException { sitexml.put(siteKey, global.getMandatoryOption(optionKey)); }
/** * Get a mandatory integer option; use {@link Integer#decode(String)} so as to take hex * oct and bin values too. * * @param option option name * @return parsed value * @throws NumberFormatException if the option could not be parsed. * @throws BadConfigException if the option could not be found */ public int getMandatoryOptionInt(String option) throws BadConfigException { getMandatoryOption(option); return getOptionInt(option, 0); }
/** * Get a mandatory integer option; use {@link Integer#decode(String)} so as to take hex * oct and bin values too. * * @param option option name * @return parsed value * @throws NumberFormatException if the option could not be parsed. * @throws BadConfigException if the option could not be found */ public int getMandatoryOptionInt(String option) throws BadConfigException { getMandatoryOption(option); return getOptionInt(option, 0); }
/** * probe to see if accumulo has already been installed. * @param cd cluster description * @return true if the relevant data directory looks inited * @throws IOException IO problems */ private boolean isInited(AggregateConf cd) throws IOException, BadConfigException { String dataDir = cd.getInternalOperations() .getGlobalOptions() .getMandatoryOption( InternalKeys.INTERNAL_DATA_DIR_PATH); Path accumuloInited = new Path(dataDir, INSTANCE_ID); FileSystem fs2 = FileSystem.get(accumuloInited.toUri(), getConf()); return fs2.exists(accumuloInited); }
.getInternalsSnapshot() .getGlobalOptions() .getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH) + dataDirSuffix); tokens.put("${JAVA_HOME}", appConf.get(AgentKeys.JAVA_HOME)); tokens.put("${COMPONENT_NAME}", componentName);
.getInternalsSnapshot() .getGlobalOptions() .getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH) + dataDirSuffix); tokens.put("${JAVA_HOME}", appConf.get(AgentKeys.JAVA_HOME)); tokens.put("${COMPONENT_NAME}", componentName);
ProviderUtils.validatePathReferencesLocalDir("ACCUMULO_HOME", accumuloPath); env.put(ACCUMULO_CONF_DIR, confDir.getAbsolutePath()); String zkHome = globalOptions.getMandatoryOption(OPTION_ZK_HOME); ProviderUtils.validatePathReferencesLocalDir("ZOOKEEPER_HOME", zkHome);
/** * Build the accumulo-site.xml file * This the configuration used by Accumulo directly * @param instanceDescription this is the cluster specification used to define this * @return a map of the dynamic bindings for this Slider instance */ public Map<String, String> buildSiteConfFromInstance( AggregateConf instanceDescription) throws BadConfigException { ConfTreeOperations appconf = instanceDescription.getAppConfOperations(); MapOperations globalAppOptions = appconf.getGlobalOptions(); MapOperations globalInstanceOptions = instanceDescription.getInternalOperations().getGlobalOptions(); Map<String, String> sitexml = new HashMap<String, String>(); providerUtils.propagateSiteOptions(globalAppOptions, sitexml); propagateClientFSBinding(sitexml); setDatabasePath(sitexml, globalInstanceOptions.getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH)); String quorum = globalAppOptions.getMandatoryOption(OptionKeys.ZOOKEEPER_QUORUM); sitexml.put(AccumuloConfigFileOptions.ZOOKEEPER_HOST, quorum); return sitexml; }
/** * return the HDFS path where the application package has been uploaded * manually or by using slider client (install package command) * * @param conf configuration * @param roleGroup name of component * @return */ public static String getApplicationDefinitionPath(ConfTreeOperations conf, String roleGroup) throws BadConfigException { String appDefPath = conf.getGlobalOptions().getMandatoryOption( AgentKeys.APP_DEF); if (roleGroup != null) { MapOperations component = conf.getComponent(roleGroup); if (component != null) { appDefPath = component.getOption(AgentKeys.APP_DEF, appDefPath); } } return appDefPath; }
@Override //Client public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem, String clustername, Configuration configuration, AggregateConf instanceDefinition, Path clusterDirPath, Path generatedConfDirPath, boolean secure) throws SliderException, IOException { super.preflightValidateClusterConfiguration(sliderFileSystem, clustername, configuration, instanceDefinition, clusterDirPath, generatedConfDirPath, secure); //add a check for the directory being writeable by the current user String dataPath = instanceDefinition.getInternalOperations() .getGlobalOptions() .getMandatoryOption( InternalKeys.INTERNAL_DATA_DIR_PATH); Path path = new Path(dataPath); sliderFileSystem.verifyDirectoryWriteAccess(path); Path historyPath = new Path(clusterDirPath, SliderKeys.HISTORY_DIR_NAME); sliderFileSystem.verifyDirectoryWriteAccess(historyPath); }
/** * return the HDFS path where the application package has been uploaded * manually or by using slider client (install package command) * * @param conf configuration * @param roleGroup name of component * @return */ public static String getApplicationDefinitionPath(ConfTreeOperations conf, String roleGroup) throws BadConfigException { String appDefPath = conf.getGlobalOptions().getMandatoryOption( AgentKeys.APP_DEF); if (roleGroup != null) { MapOperations component = conf.getComponent(roleGroup); if (component != null) { appDefPath = component.getOption(AgentKeys.APP_DEF, appDefPath); } } return appDefPath; }
@Override //Client public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem, String clustername, Configuration configuration, AggregateConf instanceDefinition, Path clusterDirPath, Path generatedConfDirPath, boolean secure) throws SliderException, IOException { super.preflightValidateClusterConfiguration(sliderFileSystem, clustername, configuration, instanceDefinition, clusterDirPath, generatedConfDirPath, secure); //add a check for the directory being writeable by the current user String dataPath = instanceDefinition.getInternalOperations() .getGlobalOptions() .getMandatoryOption( InternalKeys.INTERNAL_DATA_DIR_PATH); Path path = new Path(dataPath); sliderFileSystem.verifyDirectoryWriteAccess(path); Path historyPath = new Path(clusterDirPath, SliderKeys.HISTORY_DIR_NAME); sliderFileSystem.verifyDirectoryWriteAccess(historyPath); }
globalInstanceOptions.getMandatoryOption( InternalKeys.INTERNAL_DATA_DIR_PATH) ); sitexml, KEY_ZNODE_PARENT); String quorum = globalAppOptions.getMandatoryOption(OptionKeys.ZOOKEEPER_QUORUM); sitexml.put(KEY_ZOOKEEPER_QUORUM, ZookeeperUtils.convertToHostsOnlyList(quorum));
/** * Build a dynamic provider role * @param name name of role * @param group group of role * @return a new provider role * @throws BadConfigException bad configuration */ public ProviderRole createDynamicProviderRole(String name, String group, MapOperations component) throws BadConfigException { String priOpt = component.getMandatoryOption(COMPONENT_PRIORITY); int priority = SliderUtils.parseAndValidate( "value of " + name + " " + COMPONENT_PRIORITY, priOpt, 0, 1, -1); String placementOpt = component.getOption(COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy.DEFAULT)); int placement = SliderUtils.parseAndValidate( "value of " + name + " " + COMPONENT_PLACEMENT_POLICY, placementOpt, 0, 0, -1); int placementTimeout = component.getOptionInt(PLACEMENT_ESCALATE_DELAY, DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS); ProviderRole newRole = new ProviderRole(name, group, priority, placement, getNodeFailureThresholdForRole(group), placementTimeout, component.getOption(YARN_LABEL_EXPRESSION, DEF_YARN_LABEL_EXPRESSION)); log.info("New {} ", newRole); return newRole; }
/** * Build a dynamic provider role * @param name name of role * @param group group of role * @return a new provider role * @throws BadConfigException bad configuration */ public ProviderRole createDynamicProviderRole(String name, String group, MapOperations component) throws BadConfigException { String priOpt = component.getMandatoryOption(COMPONENT_PRIORITY); int priority = SliderUtils.parseAndValidate( "value of " + name + " " + COMPONENT_PRIORITY, priOpt, 0, 1, -1); String placementOpt = component.getOption(COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy.DEFAULT)); int placement = SliderUtils.parseAndValidate( "value of " + name + " " + COMPONENT_PLACEMENT_POLICY, placementOpt, 0, 0, -1); int placementTimeout = component.getOptionInt(PLACEMENT_ESCALATE_DELAY, DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS); ProviderRole newRole = new ProviderRole(name, group, priority, placement, getNodeFailureThresholdForRole(group), placementTimeout, component.getOption(YARN_LABEL_EXPRESSION, DEF_YARN_LABEL_EXPRESSION)); log.info("New {} ", newRole); return newRole; }
@VisibleForTesting protected void addUpgradeCommand(String roleName, String roleGroup, String containerId, HeartBeatResponse response, String scriptPath, long timeout) throws SliderException { assert getAmState().isApplicationLive(); ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot(); ExecutionCommand cmd = new ExecutionCommand( AgentCommandType.EXECUTION_COMMAND); prepareExecutionCommand(cmd); String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME); String hostName = getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME); cmd.setHostname(hostName); cmd.setClusterName(clusterName); cmd.setRoleCommand(Command.UPGRADE.toString()); cmd.setServiceName(clusterName); cmd.setComponentName(roleName); cmd.setRole(roleName); Map<String, String> hostLevelParams = new TreeMap<String, String>(); hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions() .getMandatoryOption(JAVA_HOME)); hostLevelParams.put(CONTAINER_ID, containerId); cmd.setHostLevelParams(hostLevelParams); cmd.setCommandParams(commandParametersSet(scriptPath, timeout, true)); Map<String, Map<String, String>> configurations = buildCommandConfigurations( appConf, containerId, roleName, roleGroup); cmd.setConfigurations(configurations); response.addExecutionCommand(cmd); }
@VisibleForTesting protected void addUpgradeCommand(String roleName, String roleGroup, String containerId, HeartBeatResponse response, String scriptPath, long timeout) throws SliderException { assert getAmState().isApplicationLive(); ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot(); ExecutionCommand cmd = new ExecutionCommand( AgentCommandType.EXECUTION_COMMAND); prepareExecutionCommand(cmd); String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME); String hostName = getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME); cmd.setHostname(hostName); cmd.setClusterName(clusterName); cmd.setRoleCommand(Command.UPGRADE.toString()); cmd.setServiceName(clusterName); cmd.setComponentName(roleName); cmd.setRole(roleName); Map<String, String> hostLevelParams = new TreeMap<String, String>(); hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions() .getMandatoryOption(JAVA_HOME)); hostLevelParams.put(CONTAINER_ID, containerId); cmd.setHostLevelParams(hostLevelParams); cmd.setCommandParams(commandParametersSet(scriptPath, timeout, true)); Map<String, Map<String, String>> configurations = buildCommandConfigurations( appConf, containerId, roleName, roleGroup); cmd.setConfigurations(configurations); response.addExecutionCommand(cmd); }
Map<String, String> hostLevelParamsStop = new TreeMap<String, String>(); hostLevelParamsStop.put(JAVA_HOME, appConf.getGlobalOptions() .getMandatoryOption(JAVA_HOME)); hostLevelParamsStop.put(CONTAINER_ID, containerId); cmdStop.setHostLevelParams(hostLevelParamsStop);
Map<String, String> hostLevelParamsStop = new TreeMap<String, String>(); hostLevelParamsStop.put(JAVA_HOME, appConf.getGlobalOptions() .getMandatoryOption(JAVA_HOME)); hostLevelParamsStop.put(CONTAINER_ID, containerId); cmdStop.setHostLevelParams(hostLevelParamsStop);