public void setOffSwitchPerHeartbeatLimit(int limit) { setInt(OFFSWITCH_PER_HEARTBEAT_LIMIT, limit); }
public void setGlobalMaximumApplicationsPerQueue(int val) { setInt(QUEUE_GLOBAL_MAX_APPLICATION, val); }
@VisibleForTesting public void setNodeLocalityDelay(int nodeLocalityDelay) { setInt(NODE_LOCALITY_DELAY, nodeLocalityDelay); }
@Private public void setQueuePriority(String queue, int priority) { String queuePolicyPrefix = getQueuePrefix(queue); setInt(queuePolicyPrefix + "priority", priority); }
public void setUserLimit(String queue, int userLimit) { setInt(getQueuePrefix(queue) + USER_LIMIT, userLimit); LOG.debug("here setUserLimit: queuePrefix=" + getQueuePrefix(queue) + ", userLimit=" + getUserLimit(queue)); }
public void setUserLimit(String queue, int userLimit) { setInt(getQueuePrefix(queue) + USER_LIMIT, userLimit); LOG.debug("here setUserLimit: queuePrefix=" + getQueuePrefix(queue) + ", userLimit=" + getUserLimit(queue)); }
public void setUserLimit(String queue, int userLimit) { setInt(getQueuePrefix(queue) + USER_LIMIT, userLimit); LOG.debug("here setUserLimit: queuePrefix=" + getQueuePrefix(queue) + ", userLimit=" + getUserLimit(queue)); }
private void setMaxAllocMb(CapacitySchedulerConfiguration conf, String queueName, int maxAllocMb) { String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) + MAXIMUM_ALLOCATION_MB; conf.setInt(propName, maxAllocMb); }
private void setMaxAllocMb(CapacitySchedulerConfiguration conf, String queueName, int maxAllocMb) { String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) + CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB; conf.setInt(propName, maxAllocMb); }
private void setMaxAllocVcores(CapacitySchedulerConfiguration conf, String queueName, int maxAllocVcores) { String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) + CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_VCORES; conf.setInt(propName, maxAllocVcores); }
private void setMaxAllocVcores(CapacitySchedulerConfiguration conf, String queueName, int maxAllocVcores) { String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) + CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_VCORES; conf.setInt(propName, maxAllocVcores); }
@Before public void setUp() { CapacitySchedulerConfiguration config = new CapacitySchedulerConfiguration(); config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, DominantResourceCalculator.class.getName()); conf = new CapacitySchedulerConfiguration(config); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); conf.set(CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICIES, "resource-based"); conf.set(CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICY_NAME, "resource-based"); String policyName = CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICY_NAME + ".resource-based" + ".class"; conf.set(policyName, POLICY_CLASS_NAME); conf.setBoolean(CapacitySchedulerConfiguration.MULTI_NODE_PLACEMENT_ENABLED, true); conf.setInt("yarn.scheduler.minimum-allocation-mb", 512); conf.setInt("yarn.scheduler.minimum-allocation-vcores", 1); }
@Before void setUp() throws Exception { conf = new CapacitySchedulerConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true); conf.setClass(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, ProportionalCapacityPreemptionPolicy.class, SchedulingEditPolicy.class); conf = (CapacitySchedulerConfiguration) TestUtils .getConfigurationWithMultipleQueues(this.conf); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 100 * GB); // Set preemption related configurations conf.setInt(CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, 0); conf.setFloat(CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND, 1.0f); conf.setFloat( CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR, 1.0f); conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, 60000L); mgr = new NullRMNodeLabelsManager(); mgr.init(this.conf); clock = mock(Clock.class); when(clock.getTime()).thenReturn(0L); }
@Test (timeout = 30000) public void testLocalityDelaysAfterQueueRefresh() throws Exception { // Manipulate queue 'e' LeafQueue e = stubLeafQueue((LeafQueue)queues.get(E)); // before reinitialization assertEquals(40, e.getNodeLocalityDelay()); assertEquals(-1, e.getRackLocalityAdditionalDelay()); csConf.setInt(CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY, 60); csConf.setInt( CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY, 600); Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, ROOT, newQueues, queues, TestUtils.spyHook); root.reinitialize(newRoot, cs.getClusterResource()); // after reinitialization assertEquals(60, e.getNodeLocalityDelay()); assertEquals(600, e.getRackLocalityAdditionalDelay()); }
@Test (timeout = 30000) public void testNodeLocalityAfterQueueRefresh() throws Exception { // Manipulate queue 'e' LeafQueue e = stubLeafQueue((LeafQueue)queues.get(E)); // before reinitialization assertEquals(40, e.getNodeLocalityDelay()); csConf.setInt(CapacitySchedulerConfiguration .NODE_LOCALITY_DELAY, 60); Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>(); CSQueue newRoot = CapacityScheduler.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, TestUtils.spyHook); queues = newQueues; root.reinitialize(newRoot, cs.getClusterResource()); // after reinitialization assertEquals(60, e.getNodeLocalityDelay()); }
@Test(timeout = 30000) public void testAMLimitDouble() throws Exception { CapacitySchedulerConfiguration config = new CapacitySchedulerConfiguration(); config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, DominantResourceCalculator.class.getName()); CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(config); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); conf.setInt("yarn.scheduler.minimum-allocation-mb", 512); conf.setInt("yarn.scheduler.minimum-allocation-vcores", 1); MockRM rm = new MockRM(conf); rm.start(); rm.registerNode("127.0.0.1:1234", 10 * GB); rm.registerNode("127.0.0.1:1235", 10 * GB); rm.registerNode("127.0.0.1:1236", 10 * GB); rm.registerNode("127.0.0.1:1237", 10 * GB); ResourceScheduler scheduler = rm.getRMContext().getScheduler(); waitforNMRegistered(scheduler, 4, 5); LeafQueue queueA = (LeafQueue) ((CapacityScheduler) scheduler).getQueue("default"); Resource amResourceLimit = queueA.getAMResourceLimit(); Assert.assertEquals(4096, amResourceLimit.getMemorySize()); Assert.assertEquals(4, amResourceLimit.getVirtualCores()); rm.stop(); }
@Test(timeout = 60000) public void testAssignMultipleOffswitchContainers() throws Exception { MockRM rm1 = new MockRM(); rm1.getRMContext().setNodeLabelManager(mgr); rm1.start(); MockNM nm1 = rm1.registerNode("h1:1234", 80 * GB); // launch an app to queue, AM container should be launched in nm1 RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default"); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); am1.allocate("*", 1 * GB, 5, new ArrayList<ContainerId>()); CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler(); RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId()); // Do node heartbeats once cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); FiCaSchedulerApp schedulerApp1 = cs.getApplicationAttempt(am1.getApplicationAttemptId()); // App1 will get one container allocated (plus AM container Assert.assertEquals(2, schedulerApp1.getLiveContainers().size()); // Set assign multiple off-switch containers to 3 CapacitySchedulerConfiguration newCSConf = new CapacitySchedulerConfiguration(); newCSConf.setInt( CapacitySchedulerConfiguration.OFFSWITCH_PER_HEARTBEAT_LIMIT, 3); cs.reinitialize(newCSConf, rm1.getRMContext()); // Do node heartbeats once cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); // App1 will get 3 new container allocated (plus 2 previously allocated // container) Assert.assertEquals(5, schedulerApp1.getLiveContainers().size()); rm1.close(); }
conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100); conf.setInt(CapacitySchedulerConfiguration.QUEUE_GLOBAL_MAX_APPLICATION, 20); conf.setInt("yarn.scheduler.capacity.root.a.a1.maximum-applications", 1); conf.setFloat("yarn.scheduler.capacity.root.d.user-limit-factor", 0.1f); conf.setInt("yarn.scheduler.capacity.maximum-applications", 4); final String a = CapacitySchedulerConfiguration.ROOT + ".a"; final String b = CapacitySchedulerConfiguration.ROOT + ".b";
CapacitySchedulerConfiguration.ASSIGN_MULTIPLE_ENABLED, true); csConf.setInt(CapacitySchedulerConfiguration.MAX_ASSIGN_PER_HEARTBEAT, 2); setupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(csConf);
false); newConf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); newConf.setInt(YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS, 1); newConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{Q1}); newConf.setQueues(Q1_PATH, new String[]{Q2});