/** * Get the time range of a set of keys * @param basekey base key to which suffix gets applied * @param defDays * @param defHours * @param defMins * @param defSecs * @return the aggregate time range in seconds */ public long getTimeRange(String basekey, int defDays, int defHours, int defMins, int defSecs) { Preconditions.checkArgument(basekey != null); int days = getOptionInt(basekey + DAYS, defDays); int hours = getOptionInt(basekey + HOURS, defHours); int minutes = getOptionInt(basekey + MINUTES, defMins); int seconds = getOptionInt(basekey + SECONDS, defSecs); // range check Preconditions.checkState(days >= 0 && hours >= 0 && minutes >= 0 && seconds >= 0, "Time range for %s has negative time component %s:%s:%s:%s", basekey, days, hours, minutes, seconds); // calculate total time, schedule the reset if expected long totalMinutes = (long) days * 24 * 60 + (long) hours * 24 + minutes; return totalMinutes * 60 + seconds; }
/** * Get the time range of a set of keys * @param basekey base key to which suffix gets applied * @param defDays * @param defHours * @param defMins * @param defSecs * @return the aggregate time range in seconds */ public long getTimeRange(String basekey, int defDays, int defHours, int defMins, int defSecs) { Preconditions.checkArgument(basekey != null); int days = getOptionInt(basekey + DAYS, defDays); int hours = getOptionInt(basekey + HOURS, defHours); int minutes = getOptionInt(basekey + MINUTES, defMins); int seconds = getOptionInt(basekey + SECONDS, defSecs); // range check Preconditions.checkState(days >= 0 && hours >= 0 && minutes >= 0 && seconds >= 0, "Time range for %s has negative time component %s:%s:%s:%s", basekey, days, hours, minutes, seconds); // calculate total time, schedule the reset if expected long totalMinutes = (long) days * 24 * 60 + (long) hours * 24 + minutes; return totalMinutes * 60 + seconds; }
/** * Get a mandatory integer option; use {@link Integer#decode(String)} so as to take hex * oct and bin values too. * * @param option option name * @return parsed value * @throws NumberFormatException if the option could not be parsed. * @throws BadConfigException if the option could not be found */ public int getMandatoryOptionInt(String option) throws BadConfigException { getMandatoryOption(option); return getOptionInt(option, 0); }
/** * Get a mandatory integer option; use {@link Integer#decode(String)} so as to take hex * oct and bin values too. * * @param option option name * @return parsed value * @throws NumberFormatException if the option could not be parsed. * @throws BadConfigException if the option could not be found */ public int getMandatoryOptionInt(String option) throws BadConfigException { getMandatoryOption(option); return getOptionInt(option, 0); }
/** * Update the AM resource with any local needs * @param capability capability to update */ public void prepareAMResourceRequirements(MapOperations sliderAM, Resource capability) { capability.setMemory(sliderAM.getOptionInt( ResourceKeys.YARN_MEMORY, capability.getMemory())); capability.setVirtualCores( sliderAM.getOptionInt(ResourceKeys.YARN_CORES, capability.getVirtualCores())); }
/** * Update the AM resource with any local needs * @param capability capability to update */ public void prepareAMResourceRequirements(MapOperations sliderAM, Resource capability) { capability.setMemory(sliderAM.getOptionInt( ResourceKeys.YARN_MEMORY, capability.getMemory())); capability.setVirtualCores( sliderAM.getOptionInt(ResourceKeys.YARN_CORES, capability.getVirtualCores())); }
private long getContainerReleaseTimeout() { // Get container release timeout in millis or 0 if the property is not set. // If non-zero then add the agent heartbeat delay time, since it can take up // to that much time for agents to receive the stop command. int timeout = getInstanceDefinition().getAppConfOperations() .getGlobalOptions() .getOptionInt(SliderKeys.APP_CONTAINER_RELEASE_TIMEOUT, 0); if (timeout > 0) { timeout += SliderKeys.APP_CONTAINER_HEARTBEAT_INTERVAL_SEC; } // convert to millis long timeoutInMillis = timeout * 1000l; log.info("Container release timeout in millis = {}", timeoutInMillis); return timeoutInMillis; }
private long getContainerReleaseTimeout() { // Get container release timeout in millis or 0 if the property is not set. // If non-zero then add the agent heartbeat delay time, since it can take up // to that much time for agents to receive the stop command. int timeout = getInstanceDefinition().getAppConfOperations() .getGlobalOptions() .getOptionInt(SliderKeys.APP_CONTAINER_RELEASE_TIMEOUT, 0); if (timeout > 0) { timeout += SliderKeys.APP_CONTAINER_HEARTBEAT_INTERVAL_SEC; } // convert to millis long timeoutInMillis = timeout * 1000l; log.info("Container release timeout in millis = {}", timeoutInMillis); return timeoutInMillis; }
public void validateNodeCount(AggregateConf instanceDescription, String name, int min, int max) throws BadCommandArgumentsException { MapOperations component = instanceDescription.getResourceOperations().getComponent(name); int count; if (component == null) { count = 0; } else { count = component.getOptionInt(ResourceKeys.COMPONENT_INSTANCES, 0); } validateNodeCount(name, count, min, max); }
public void validateNodeCount(AggregateConf instanceDescription, String name, int min, int max) throws BadCommandArgumentsException { MapOperations component = instanceDescription.getResourceOperations().getComponent(name); int count; if (component == null) { count = 0; } else { count = component.getOptionInt(ResourceKeys.COMPONENT_INSTANCES, 0); } validateNodeCount(name, count, min, max); }
/** * Extract any resource requirements from this component's settings. * All fields that are set will override the existing values -if * unset that resource field will be left unchanged. * * Important: the configuration must already be fully resolved * in order to pick up global options. * @param resource resource to configure * @param map map of options */ public void extractResourceRequirements(Resource resource, Map<String, String> map) { if (map != null) { MapOperations options = new MapOperations("", map); resource.setMemory(options.getOptionInt(ResourceKeys.YARN_MEMORY, resource.getMemory())); resource.setVirtualCores(options.getOptionInt(ResourceKeys.YARN_CORES, resource.getVirtualCores())); } }
/** * Extract any resource requirements from this component's settings. * All fields that are set will override the existing values -if * unset that resource field will be left unchanged. * * Important: the configuration must already be fully resolved * in order to pick up global options. * @param resource resource to configure * @param map map of options */ public void extractResourceRequirements(Resource resource, Map<String, String> map) { if (map != null) { MapOperations options = new MapOperations("", map); resource.setMemory(options.getOptionInt(ResourceKeys.YARN_MEMORY, resource.getMemory())); resource.setVirtualCores(options.getOptionInt(ResourceKeys.YARN_CORES, resource.getVirtualCores())); } }
/** * Verify that an instance definition is considered valid by the provider * @param instanceDefinition instance definition * @throws SliderException if the configuration is not valid */ public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws SliderException { super.validateInstanceDefinition(instanceDefinition, fs); // make sure there is no negative entry in the instance count Map<String, Map<String, String>> instanceMap = instanceDefinition.getResources().components; for (Map.Entry<String, Map<String, String>> entry : instanceMap.entrySet()) { MapOperations mapOperations = new MapOperations(entry); int instances = mapOperations.getOptionInt(COMPONENT_INSTANCES, 0); if (instances < 0) { throw new BadClusterStateException( "Component %s has negative instance count: %d", mapOperations.name, instances); } } }
/** * Verify that an instance definition is considered valid by the provider * @param instanceDefinition instance definition * @throws SliderException if the configuration is not valid */ public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws SliderException { super.validateInstanceDefinition(instanceDefinition, fs); // make sure there is no negative entry in the instance count Map<String, Map<String, String>> instanceMap = instanceDefinition.getResources().components; for (Map.Entry<String, Map<String, String>> entry : instanceMap.entrySet()) { MapOperations mapOperations = new MapOperations(entry); int instances = mapOperations.getOptionInt(COMPONENT_INSTANCES, 0); if (instances < 0) { throw new BadClusterStateException( "Component %s has negative instance count: %d", mapOperations.name, instances); } } }
/** * Schedule the escalation action * @param internal * @throws BadConfigException */ private void scheduleEscalation(ConfTree internal) throws BadConfigException { EscalateOutstandingRequests escalate = new EscalateOutstandingRequests(); ConfTreeOperations ops = new ConfTreeOperations(internal); int seconds = ops.getGlobalOptions().getOptionInt(InternalKeys.ESCALATION_CHECK_INTERVAL, InternalKeys.DEFAULT_ESCALATION_CHECK_INTERVAL); RenewingAction<EscalateOutstandingRequests> renew = new RenewingAction<>( escalate, seconds, seconds, TimeUnit.SECONDS, 0); actionQueues.renewing("escalation", renew); }
/** * Schedule the escalation action * @param internal * @throws BadConfigException */ private void scheduleEscalation(ConfTree internal) throws BadConfigException { EscalateOutstandingRequests escalate = new EscalateOutstandingRequests(); ConfTreeOperations ops = new ConfTreeOperations(internal); int seconds = ops.getGlobalOptions().getOptionInt(InternalKeys.ESCALATION_CHECK_INTERVAL, InternalKeys.DEFAULT_ESCALATION_CHECK_INTERVAL); RenewingAction<EscalateOutstandingRequests> renew = new RenewingAction<>( escalate, seconds, seconds, TimeUnit.SECONDS, 0); actionQueues.renewing("escalation", renew); }
int amLaunchFailProbability = internals.getOptionInt( InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_LAUNCH_FAILURE, 0); int amKillProbability = internals.getOptionInt( InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE, InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE); monkey.addTarget("AM killer", new ChaosKillAM(actionQueues, -1), amKillProbability); int containerKillProbability = internals.getOptionInt( InternalKeys.CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE, InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE);
/** * Build a dynamic provider role * @param name name of role * @param group group of role * @return a new provider role * @throws BadConfigException bad configuration */ public ProviderRole createDynamicProviderRole(String name, String group, MapOperations component) throws BadConfigException { String priOpt = component.getMandatoryOption(COMPONENT_PRIORITY); int priority = SliderUtils.parseAndValidate( "value of " + name + " " + COMPONENT_PRIORITY, priOpt, 0, 1, -1); String placementOpt = component.getOption(COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy.DEFAULT)); int placement = SliderUtils.parseAndValidate( "value of " + name + " " + COMPONENT_PLACEMENT_POLICY, placementOpt, 0, 0, -1); int placementTimeout = component.getOptionInt(PLACEMENT_ESCALATE_DELAY, DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS); ProviderRole newRole = new ProviderRole(name, group, priority, placement, getNodeFailureThresholdForRole(group), placementTimeout, component.getOption(YARN_LABEL_EXPRESSION, DEF_YARN_LABEL_EXPRESSION)); log.info("New {} ", newRole); return newRole; }
/** * Build a dynamic provider role * @param name name of role * @param group group of role * @return a new provider role * @throws BadConfigException bad configuration */ public ProviderRole createDynamicProviderRole(String name, String group, MapOperations component) throws BadConfigException { String priOpt = component.getMandatoryOption(COMPONENT_PRIORITY); int priority = SliderUtils.parseAndValidate( "value of " + name + " " + COMPONENT_PRIORITY, priOpt, 0, 1, -1); String placementOpt = component.getOption(COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy.DEFAULT)); int placement = SliderUtils.parseAndValidate( "value of " + name + " " + COMPONENT_PLACEMENT_POLICY, placementOpt, 0, 0, -1); int placementTimeout = component.getOptionInt(PLACEMENT_ESCALATE_DELAY, DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS); ProviderRole newRole = new ProviderRole(name, group, priority, placement, getNodeFailureThresholdForRole(group), placementTimeout, component.getOption(YARN_LABEL_EXPRESSION, DEF_YARN_LABEL_EXPRESSION)); log.info("New {} ", newRole); return newRole; }
agentServer.setThreadPool( new QueuedThreadPool( configsMap.getOptionInt("agent.threadpool.size.max", 25))); agentServer.setStopAtShutdown(true); agentServer.setGracefulShutdown(1000);