/** * Parses a resource config value in one of three forms: * <ol> * <li>Percentage: "50%" or "40% memory, 60% cpu"</li> * <li>New style resources: "vcores=10, memory-mb=1024" * or "vcores=60%, memory-mb=40%"</li> * <li>Old style resources: "1024 mb, 10 vcores"</li> * </ol> * In new style resources, any resource that is not specified will be * set to {@link Long#MAX_VALUE} or 100%, as appropriate. Also, in the new * style resources, units are not allowed. Units are assumed from the resource * manager's settings for the resources when the value isn't a percentage. * * @param value the resource definition to parse * @return a {@link ConfigurableResource} that represents the parsed value * @throws AllocationConfigurationException if the raw value is not a valid * resource definition */ public static ConfigurableResource parseResourceConfigValue(String value) throws AllocationConfigurationException { return parseResourceConfigValue(value, Long.MAX_VALUE); }
public String getEventlogDir() { return get(EVENT_LOG_DIR, new File(System.getProperty("hadoop.log.dir", "/tmp/")).getAbsolutePath() + File.separator + "fairscheduler"); }
public boolean getUsePortForNodeName() { return getBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME); }
private void initScheduler(Configuration conf) throws IOException { synchronized (this) { this.conf = new FairSchedulerConfiguration(conf); validateConf(this.conf); minimumAllocation = this.conf.getMinimumAllocation(); initMaximumResourceCapability(this.conf.getMaximumAllocation()); incrAllocation = this.conf.getIncrementAllocation(); continuousSchedulingEnabled = this.conf.isContinuousSchedulingEnabled(); continuousSchedulingSleepMs = this.conf.getContinuousSchedulingSleepMs(); nodeLocalityThreshold = this.conf.getLocalityThresholdNode(); rackLocalityThreshold = this.conf.getLocalityThresholdRack(); nodeLocalityDelayMs = this.conf.getLocalityDelayNodeMs(); rackLocalityDelayMs = this.conf.getLocalityDelayRackMs(); preemptionEnabled = this.conf.getPreemptionEnabled(); preemptionUtilizationThreshold = this.conf.getPreemptionUtilizationThreshold(); assignMultiple = this.conf.getAssignMultiple(); maxAssign = this.conf.getMaxAssign(); sizeBasedWeight = this.conf.getSizeBasedWeight(); preemptionInterval = this.conf.getPreemptionInterval(); waitTimeBeforeKill = this.conf.getWaitTimeBeforeKill(); usePortForNodeName = this.conf.getUsePortForNodeName(); updateInterval = this.conf.getUpdateInterval(); if (updateInterval < 0) { updateInterval = FairSchedulerConfiguration.DEFAULT_UPDATE_INTERVAL_MS;
writeLock.lock(); try { this.conf = new FairSchedulerConfiguration(conf); validateConf(this.conf); authorizer = YarnAuthorizationProvider.getInstance(conf); minimumAllocation = super.getMinimumAllocation(); initMaximumResourceCapability(super.getMaximumAllocation()); incrAllocation = this.conf.getIncrementAllocation(); updateReservationThreshold(); continuousSchedulingEnabled = this.conf.isContinuousSchedulingEnabled(); continuousSchedulingSleepMs = this.conf.getContinuousSchedulingSleepMs(); nodeLocalityThreshold = this.conf.getLocalityThresholdNode(); rackLocalityThreshold = this.conf.getLocalityThresholdRack(); nodeLocalityDelayMs = this.conf.getLocalityDelayNodeMs(); rackLocalityDelayMs = this.conf.getLocalityDelayRackMs(); assignMultiple = this.conf.getAssignMultiple(); maxAssignDynamic = this.conf.isMaxAssignDynamic(); maxAssign = this.conf.getMaxAssign(); sizeBasedWeight = this.conf.getSizeBasedWeight(); usePortForNodeName = this.conf.getUsePortForNodeName(); reservableNodesRatio = this.conf.getReservableNodes(); updateInterval = this.conf.getUpdateInterval(); if (updateInterval < 0) { updateInterval = FairSchedulerConfiguration.DEFAULT_UPDATE_INTERVAL_MS; if (this.conf.getPreemptionEnabled()) { createPreemptionThread();
@Test(expected=IllegalArgumentException.class) public void testAllocationIncrementInvalidUnit() throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RESOURCE_TYPES + "." + ResourceInformation.MEMORY_MB.getName() + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "1 Xi"); new FairSchedulerConfiguration(conf).getIncrementAllocation(); }
@SuppressWarnings("deprecation") FSPreemptionThread(FairScheduler scheduler) { setDaemon(true); setName("FSPreemptionThread"); this.scheduler = scheduler; this.context = scheduler.getContext(); FairSchedulerConfiguration fsConf = scheduler.getConf(); context.setPreemptionEnabled(); context.setPreemptionUtilizationThreshold( fsConf.getPreemptionUtilizationThreshold()); preemptionTimer = new Timer("Preemption Timer", true); warnTimeBeforeKill = fsConf.getWaitTimeBeforeKill(); long allocDelay = (fsConf.isContinuousSchedulingEnabled() ? 10 * fsConf.getContinuousSchedulingSleepMs() // 10 runs : 4 * scheduler.getNMHeartbeatInterval()); // 4 heartbeats delayBeforeNextStarvationCheck = warnTimeBeforeKill + allocDelay + fsConf.getWaitTimeBeforeNextStarvationCheck(); schedulerReadLock = scheduler.getSchedulerReadLock(); }
@Before public void setUp() throws Exception { scheduler = new FairScheduler(); conf = new FairSchedulerConfiguration(); }
for (int i=0; i < resourceTypes.length; ++i) { String name = resourceTypes[i].getName(); String propertyKey = getAllocationIncrementPropKey(name); String propValue = get(propertyKey); if (propValue != null) { Matcher matcher = RESOURCE_REQUEST_VALUE_PATTERN.matcher(propValue); long value = Long.parseLong(matcher.group(1)); String unit = matcher.group(2); long valueInDefaultUnits = getValueInDefaultUnits(value, unit, name); others.put(name, valueInDefaultUnits); } else { if (get(RM_SCHEDULER_INCREMENT_ALLOCATION_MB) != null) { String overridingKey = getAllocationIncrementPropKey( ResourceInformation.MEMORY_MB.getName()); LOG.warn("Configuration " + overridingKey + "=" + get(overridingKey) + " is overriding the " + RM_SCHEDULER_INCREMENT_ALLOCATION_MB + "=" + get(RM_SCHEDULER_INCREMENT_ALLOCATION_MB) + " property"); memory = getLong( RM_SCHEDULER_INCREMENT_ALLOCATION_MB, DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB); if (get(RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES) != null) { String overridingKey = getAllocationIncrementPropKey( ResourceInformation.VCORES.getName()); LOG.warn("Configuration " + overridingKey + "=" + get(overridingKey) + " is overriding the " + RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES +
public int getMaxAssign() { return getInt(MAX_ASSIGN, DEFAULT_MAX_ASSIGN); }
@Override protected Configuration createConfiguration() throws IOException { FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration(); out.println("</allocations>"); out.close(); fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); fsConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); fsConf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
public float getReservationThresholdIncrementMultiple() { return getFloat( RM_SCHEDULER_RESERVATION_THRESHOLD_INCREMENT_MULTIPLE, DEFAULT_RM_SCHEDULER_RESERVATION_THRESHOLD_INCREMENT_MULTIPLE); }
private void validateConf(FairSchedulerConfiguration config) { config.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); int maxMem = config.getInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); long incrementMem = config.getIncrementAllocation().getMemorySize(); if (incrementMem <= 0) { throw new YarnRuntimeException("Invalid resource scheduler memory" config.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); int maxVcores = config.getInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); int incrementVcore = config.getIncrementAllocation().getVirtualCores(); if (incrementVcore <= 0) { throw new YarnRuntimeException("Invalid resource scheduler vcores"
@SuppressWarnings("deprecation") private void triggerSchedulingAttempt() throws InterruptedException { Thread.sleep( 2 * scheduler.getConf().getContinuousSchedulingSleepMs()); } }
private static ConfigurableResource parseOldStyleResource(String value) throws AllocationConfigurationException { final String lCaseValue = StringUtils.toLowerCase(value); int memory = findResource(lCaseValue, "mb"); int vcores = findResource(lCaseValue, "vcores"); return new ConfigurableResource( BuilderUtils.newResource(memory, vcores)); }
private void initScheduler(Configuration conf) throws IOException { synchronized (this) { this.conf = new FairSchedulerConfiguration(conf); validateConf(this.conf); minimumAllocation = this.conf.getMinimumAllocation(); initMaximumResourceCapability(this.conf.getMaximumAllocation()); incrAllocation = this.conf.getIncrementAllocation(); continuousSchedulingEnabled = this.conf.isContinuousSchedulingEnabled(); continuousSchedulingSleepMs = this.conf.getContinuousSchedulingSleepMs(); nodeLocalityThreshold = this.conf.getLocalityThresholdNode(); rackLocalityThreshold = this.conf.getLocalityThresholdRack(); nodeLocalityDelayMs = this.conf.getLocalityDelayNodeMs(); rackLocalityDelayMs = this.conf.getLocalityDelayRackMs(); preemptionEnabled = this.conf.getPreemptionEnabled(); preemptionUtilizationThreshold = this.conf.getPreemptionUtilizationThreshold(); assignMultiple = this.conf.getAssignMultiple(); maxAssign = this.conf.getMaxAssign(); sizeBasedWeight = this.conf.getSizeBasedWeight(); preemptionInterval = this.conf.getPreemptionInterval(); waitTimeBeforeKill = this.conf.getWaitTimeBeforeKill(); usePortForNodeName = this.conf.getUsePortForNodeName(); updateInterval = this.conf.getUpdateInterval(); if (updateInterval < 0) { updateInterval = FairSchedulerConfiguration.DEFAULT_UPDATE_INTERVAL_MS;
@Test public void testMemoryIncrementConfiguredViaMultipleProperties() { TestAppender testAppender = new TestAppender(); Log4JLogger logger = (Log4JLogger) FairSchedulerConfiguration.LOG; logger.getLogger().addAppender(testAppender); try { Configuration conf = new Configuration(); conf.set("yarn.scheduler.increment-allocation-mb", "7"); conf.set(YarnConfiguration.RESOURCE_TYPES + "." + ResourceInformation.MEMORY_MB.getName() + FairSchedulerConfiguration.INCREMENT_ALLOCATION, "13"); FairSchedulerConfiguration fsc = new FairSchedulerConfiguration(conf); Resource increment = fsc.getIncrementAllocation(); Assert.assertEquals(13L, increment.getMemorySize()); assertTrue("Warning message is not logged when specifying memory " + "increment via multiple properties", testAppender.getLogEvents().stream().anyMatch( e -> e.getLevel() == Level.WARN && ("Configuration " + "yarn.resource-types.memory-mb.increment-allocation=13 is " + "overriding the yarn.scheduler.increment-allocation-mb=7 " + "property").equals(e.getMessage()))); } finally { logger.getLogger().removeAppender(testAppender); } }
@Before public void setup() throws Exception { Configuration conf = new Configuration(); clock = new TestFairScheduler.MockClock(); scheduler = mock(FairScheduler.class); when(scheduler.getConf()).thenReturn( new FairSchedulerConfiguration(conf)); when(scheduler.getClock()).thenReturn(clock); AllocationConfiguration allocConf = new AllocationConfiguration( conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); queueManager = new QueueManager(scheduler); queueManager.initialize(conf); queueMaxApps = allocConf.queueMaxApps; userMaxApps = allocConf.userMaxApps; maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler); appNum = 0; rmContext = mock(RMContext.class); when(rmContext.getEpoch()).thenReturn(0L); }
public int getWaitTimeBeforeKill() { return getInt(WAIT_TIME_BEFORE_KILL, DEFAULT_WAIT_TIME_BEFORE_KILL); }
public float getPreemptionUtilizationThreshold() { return getFloat(PREEMPTION_THRESHOLD, DEFAULT_PREEMPTION_THRESHOLD); }