/** * Merge in another tree with overwrites * @param that the other tree */ public void merge(ConfTree that) { getGlobalOptions().putAll(that.global); confTree.metadata.putAll(that.metadata); confTree.credentials.putAll(that.credentials); for (Map.Entry<String, Map<String, String>> entry : that.components.entrySet()) { MapOperations comp = getOrAddComponent(entry.getKey()); comp.putAll(entry.getValue()); } }
@VisibleForTesting protected List<String> getSystemConfigurationsRequested(ConfTreeOperations appConf) { List<String> configList = new ArrayList<String>(); String configTypes = appConf.get(AgentKeys.SYSTEM_CONFIGS); if (configTypes != null && configTypes.length() > 0) { String[] configs = configTypes.split(","); for (String config : configs) { configList.add(config.trim()); } } return new ArrayList<String>(new HashSet<String>(configList)); }
protected void validateComponentProperties(ConfTreeOperations props) throws BadConfigException { for (String compName : props.getComponentNames()) { MapOperations mo = props.getComponent(compName); if (mo == null) continue; for (String key : mo.keySet()) { validatePropertyNamePrefix(key); } } }
Map<String, Object> md = internalOps.getConfTree().metadata; long time = System.currentTimeMillis(); md.put(StatusKeys.INFO_CREATE_TIME_HUMAN, SliderUtils.toGMTString(time)); md.put(StatusKeys.INFO_CREATE_TIME_MILLIS, Long.toString(time)); MapOperations globalOptions = internalOps.getGlobalOptions(); BuildHelper.addBuildMetadata(md, "create"); SliderUtils.setInfoTime(md, System.currentTimeMillis()); internalOps.set(INTERNAL_AM_TMP_DIR, instancePaths.tmpPathAM.toUri()); internalOps.set(INTERNAL_TMP_DIR, instancePaths.tmpPath.toUri()); internalOps.set(INTERNAL_SNAPSHOT_CONF_PATH, instancePaths.snapshotConfPath.toUri()); internalOps.set(INTERNAL_GENERATED_CONF_PATH, instancePaths.generatedConfPath.toUri()); internalOps.set(INTERNAL_DATA_DIR_PATH, instancePaths.dataPath.toUri()); internalOps.set(INTERNAL_APPDEF_DIR_PATH, instancePaths.appDefPath.toUri()); internalOps.set(INTERNAL_ADDONS_DIR_PATH, instancePaths.addonsPath.toUri()); internalOps.set(InternalKeys.INTERNAL_PROVIDER_NAME, provider); internalOps.set(OptionKeys.APPLICATION_NAME, clustername);
for (Entry<String, Map<String, String>> entry : resources.getComponents() .entrySet()) { if (COMPONENT_AM.equals(entry.getKey())) { String externalAppDef = componentAppConf.get(AgentKeys.APP_DEF); if (SliderUtils.isSet(externalAppDef)) { Path newAppDef = new Path(coreFS.buildAppDefDirPath(clustername), component + "_" + SliderKeys.DEFAULT_APP_PKG); componentAppConf.set(AgentKeys.APP_DEF, newAppDef); componentAppConf.append(AgentKeys.APP_DEF_ORIGINAL, externalAppDef); log.info("Copying external appdef {} to {} for {}", externalAppDef, newAppDef, component); .getComponentNames()) { if (COMPONENT_AM.equals(rcomp)) { continue; componentAppConf.getOrAddComponent(rcomp); .getComponentNames()) { if (COMPONENT_AM.equals(rcomp)) { continue; String componentAppDef = componentAppConf.getComponentOpt( rcomp, AgentKeys.APP_DEF, null); if (SliderUtils.isUnset(componentAppDef) || externalAppDefs.get(componentAppDef), rcomp); } else {
getGlobalOptions().getOption(AgentKeys.AGENT_CONF, ""); if (StringUtils.isNotEmpty(agentConf)) { sliderFileSystem.verifyFileExists(new Path(agentConf)); getGlobalOptions().get(AgentKeys.PACKAGE_PATH); if (SliderUtils.isUnset(appHome)) { String agentImage = instanceDefinition.getInternalOperations(). get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH); sliderFileSystem.verifyFileExists(new Path(agentImage));
/** * Is this app package versioned? * * @return true if {@link SliderKeys#APP_VERSION} was set in the app config * provided during creation of this app * @since 0.80.0-incubating */ public boolean isVersioned() { return StringUtils.isNotEmpty(getAppConfOperations().getGlobalOptions() .get(SliderKeys.APP_VERSION)); }
private Map<String, String> getStandardTokenMap(ConfTreeOperations appConf, String componentName, String componentGroup) throws SliderException { Map<String, String> tokens = new HashMap<String, String>(); String nnuri = appConf.get("site.fs.defaultFS"); tokens.put("${NN_URI}", nnuri); tokens.put("${NN_HOST}", URI.create(nnuri).getHost()); tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS)); tokens.put("${DEFAULT_ZK_PATH}", appConf.get(OptionKeys.ZOOKEEPER_PATH)); String prefix = appConf.getComponentOpt(componentGroup, ROLE_PREFIX, null); String dataDirSuffix = ""; .getGlobalOptions() .getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH) + dataDirSuffix); tokens.put("${JAVA_HOME}", appConf.get(AgentKeys.JAVA_HOME)); tokens.put("${COMPONENT_NAME}", componentName); tokens.put("${COMPONENT_NAME.lc}", componentName.toLowerCase());
for (String component : instanceDefinition.getResourceOperations().getComponentNames()) { instanceDefinition.getAppConfOperations().getOrAddComponent(component); if (resources.getComponent(SliderKeys.COMPONENT_AM) != null) { resources.setComponentOpt( SliderKeys.COMPONENT_AM, COMPONENT_INSTANCES, "1"); resourcesSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getResources()); appConfSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getAppConf()); internalsSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getInternal());
Set<String> roleNames = resources.getComponentNames(); for (String name : roleNames) { if (roles.containsKey(name)) { MapOperations resComponent = resources.getComponent(name); ProviderRole dynamicRole = createDynamicProviderRole(name, resComponent); buildRole(dynamicRole); MapOperations globalResOpts = instanceDefinition.getResourceOperations().getGlobalOptions();
instanceDefinition.getResourceOperations(); Set<String> names = resources.getComponentNames(); names.remove(SliderKeys.COMPONENT_AM); for (String name : names) { MapOperations componentConfig = resources.getMandatoryComponent(name); int count = componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
/** * Propagate all global keys matching a prefix * @param src source * @param prefix prefix */ public void propagateGlobalKeys(ConfTree src, String prefix) { Map<String, String> global = src.global; for (Map.Entry<String, String> entry : global.entrySet()) { String key = entry.getKey(); if (key.startsWith(prefix)) { set(key, entry.getValue()); } } }
treeOps.getGlobalOptions().put(AgentKeys.JAVA_HOME, "java_home"); treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE"); treeOps.set("site.fs.defaultFS", "hdfs://HOST1:8020/"); treeOps.set("internal.data.dir.path", "hdfs://HOST1:8020/database"); treeOps.set(OptionKeys.ZOOKEEPER_HOSTS, "HOST1");
@Override public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws SliderException { super.validateInstanceDefinition(instanceDefinition, fs); ConfTreeOperations resources = instanceDefinition.getResourceOperations(); Set<String> unknownRoles = resources.getComponentNames(); unknownRoles.removeAll(knownRoleNames); if (!unknownRoles.isEmpty()) { throw new BadCommandArgumentsException("There is unknown role: %s", unknownRoles.iterator().next()); } providerUtils.validateNodeCount(instanceDefinition, AccumuloKeys.ROLE_TABLET, 1, -1); providerUtils.validateNodeCount(instanceDefinition, AccumuloKeys.ROLE_MASTER, 1, -1); providerUtils.validateNodeCount(instanceDefinition, AccumuloKeys.ROLE_GARBAGE_COLLECTOR, 0, -1); providerUtils.validateNodeCount(instanceDefinition, AccumuloKeys.ROLE_MONITOR, 0, -1); providerUtils.validateNodeCount(instanceDefinition, AccumuloKeys.ROLE_TRACER , 0, -1); MapOperations globalAppConfOptions = instanceDefinition.getAppConfOperations().getGlobalOptions(); globalAppConfOptions.verifyOptionSet(AccumuloKeys.OPTION_ZK_HOME); globalAppConfOptions.verifyOptionSet(AccumuloKeys.OPTION_HADOOP_HOME); }
@Override public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem, Configuration serviceConf, AbstractLauncher launcher, AggregateConf instanceDefinition, Path snapshotConfDirPath, Path generatedConfDirPath, Configuration clientConfExtras, String libdir, Path tempPath, boolean miniClusterTestRun) throws IOException, SliderException { String agentImage = instanceDefinition.getInternalOperations(). get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH); if (SliderUtils.isUnset(agentImage)) { Path agentPath = new Path(tempPath.getParent(), AgentKeys.PROVIDER_AGENT); log.info("Automatically uploading the agent tarball at {}", agentPath); fileSystem.getFileSystem().mkdirs(agentPath); if (ProviderUtils.addAgentTar(this, AGENT_TAR, fileSystem, agentPath)) { instanceDefinition.getInternalOperations().set( InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, new Path(agentPath, AGENT_TAR).toUri()); } } }
private void mergeExternalComponent(ConfTreeOperations ops, ConfTreeOperations externalOps, String externalComponent, Integer priority) throws BadConfigException { for (String subComponent : externalOps.getComponentNames()) { if (COMPONENT_AM.equals(subComponent)) { continue; log.debug("Merging options for {} into {}", subComponent, prefix + subComponent); MapOperations subComponentOps = ops.getOrAddComponent( prefix + subComponent); if (priority == null) { SliderUtils.mergeMaps(subComponentOps, ops.getComponent(externalComponent).options); subComponentOps.remove(COMPONENT_TYPE_KEY); externalOps.getComponent(subComponent), SliderKeys.COMPONENT_KEYS_TO_SKIP);
/** * Schedule the escalation action * @param internal * @throws BadConfigException */ private void scheduleEscalation(ConfTree internal) throws BadConfigException { EscalateOutstandingRequests escalate = new EscalateOutstandingRequests(); ConfTreeOperations ops = new ConfTreeOperations(internal); int seconds = ops.getGlobalOptions().getOptionInt(InternalKeys.ESCALATION_CHECK_INTERVAL, InternalKeys.DEFAULT_ESCALATION_CHECK_INTERVAL); RenewingAction<EscalateOutstandingRequests> renew = new RenewingAction<>( escalate, seconds, seconds, TimeUnit.SECONDS, 0); actionQueues.renewing("escalation", renew); }
ConfTreeOperations ops = new ConfTreeOperations(resources); for (String roleGroup : ops.getComponentNames()) { if (roleGroup.equals(SliderKeys.COMPONENT_AM)) { continue; long window = ops.getComponentOptLong(roleGroup, ResourceKeys.CONTAINER_HEALTH_THRESHOLD_WINDOW_SEC, ResourceKeys.DEFAULT_CONTAINER_HEALTH_THRESHOLD_WINDOW_SEC); long initDelay = ops.getComponentOptLong(roleGroup, ResourceKeys.CONTAINER_HEALTH_THRESHOLD_INIT_DELAY_SEC, ResourceKeys.DEFAULT_CONTAINER_HEALTH_THRESHOLD_INIT_DELAY_SEC); long pollFrequency = ops.getComponentOptLong(roleGroup, ResourceKeys.CONTAINER_HEALTH_THRESHOLD_POLL_FREQUENCY_SEC, ResourceKeys.DEFAULT_CONTAINER_HEALTH_THRESHOLD_POLL_FREQUENCY_SEC);
public void setResources(ConfTree resources) { this.resources = resources; resourceOperations = new ConfTreeOperations(resources); }
instanceDescription.getAppConfOperations(); MapOperations globalAppOptions = appconf.getGlobalOptions(); MapOperations globalInstanceOptions = instanceDescription.getInternalOperations().getGlobalOptions(); MapOperations master = appconf.getMandatoryComponent(HBaseKeys.ROLE_MASTER); MapOperations worker = appconf.getMandatoryComponent(HBaseKeys.ROLE_WORKER);