protected List<UserGroupMappingPlacementRule.QueueMapping> setupQueueMapping( CapacityScheduler newCS, String user, String parentQueue, String queue) { List<UserGroupMappingPlacementRule.QueueMapping> queueMappings = new ArrayList<>(); queueMappings.add(new UserGroupMappingPlacementRule.QueueMapping( UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, user, getQueueMapping(parentQueue, queue))); newCS.getConfiguration().setQueueMappings(queueMappings); return queueMappings; }
@Before public void setUp() throws Exception { super.setUp(); policy.init(cs.getConfiguration(), cs.getRMContext(), cs); }
protected MockRM setupSchedulerInstance() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); setupQueueMappings(conf, PARENT_QUEUE, true, new int[] {0, 1, 2, 3}); RMNodeLabelsManager mgr = setupNodeLabelManager(conf); MockRM newMockRM = new MockRM(conf) { protected RMNodeLabelsManager createNodeLabelManager() { return mgr; } }; newMockRM.start(); ((CapacityScheduler) newMockRM.getResourceScheduler()).start(); setupNodes(newMockRM); return newMockRM; }
private CapacitySchedulerConfiguration getSchedulerAutoCreatedQueueConfiguration( boolean overrideWithQueueMappings) throws IOException { CapacitySchedulerConfiguration schedulerConf = new CapacitySchedulerConfiguration(conf); TestCapacitySchedulerAutoCreatedQueueBase .setupQueueConfigurationForSingleAutoCreatedLeafQueue(schedulerConf); TestCapacitySchedulerAutoCreatedQueueBase.setupQueueMappings(schedulerConf, "c", overrideWithQueueMappings, new int[] {0, 1}); return schedulerConf; }
protected void validateInitialQueueEntitlement( CapacityScheduler capacityScheduler, CSQueue parentQueue, String leafQueueName, Map<String, Float> expectedTotalChildQueueAbsCapacityByLabel, Set<String> nodeLabels) throws SchedulerDynamicEditException, InterruptedException { ManagedParentQueue autoCreateEnabledParentQueue = (ManagedParentQueue) parentQueue; GuaranteedOrZeroCapacityOverTimePolicy policy = (GuaranteedOrZeroCapacityOverTimePolicy) autoCreateEnabledParentQueue .getAutoCreatedQueueManagementPolicy(); AutoCreatedLeafQueue leafQueue = (AutoCreatedLeafQueue) capacityScheduler.getQueue(leafQueueName); Map<String, QueueEntitlement> expectedEntitlements = new HashMap<>(); QueueCapacities cap = autoCreateEnabledParentQueue.getLeafQueueTemplate() .getQueueCapacities(); for (String label : nodeLabels) { validateCapacitiesByLabel(autoCreateEnabledParentQueue, leafQueue, label); assertEquals(true, policy.isActive(leafQueue, label)); assertEquals(expectedTotalChildQueueAbsCapacityByLabel.get(label), policy.getAbsoluteActivatedChildQueueCapacity(label), EPSILON); QueueEntitlement expectedEntitlement = new QueueEntitlement( cap.getCapacity(label), cap.getMaximumCapacity(label)); expectedEntitlements.put(label, expectedEntitlement); validateEffectiveMinResource(leafQueue, label, expectedEntitlements); } }
public static CapacitySchedulerConfiguration setupQueueConfigurationForSimpleSurgicalPreemption( CapacitySchedulerConfiguration conf) { //set up auto created queue configs TestCapacitySchedulerAutoCreatedQueueBase.setupQueueMappings(conf, "c", true, new int[] { 1, 2 }); //setup new queues with one of them auto enabled // Define top-level queues // Set childQueue for root conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "c" }); conf.setCapacity(C, 100f); conf.setUserLimitFactor(C, 1.0f); conf.setAutoCreateChildQueueEnabled(C, true); //Setup leaf queue template configs conf.setAutoCreatedLeafQueueConfigCapacity(C, 30.0f); conf.setAutoCreatedLeafQueueConfigMaxCapacity(C, 100.0f); conf.setAutoCreatedLeafQueueConfigUserLimit(C, 100); conf.setAutoCreatedLeafQueueConfigUserLimitFactor(C, 3.0f); return conf; }
@Test public void testPlaceApplicationWithPlacementRuleChain() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
protected void cleanupQueue(String queueName) throws YarnException { AutoCreatedLeafQueue queue = (AutoCreatedLeafQueue) cs.getQueue(queueName); if (queue != null) { setEntitlement(queue, new QueueEntitlement(0.0f, 0.0f)); ((ManagedParentQueue) queue.getParent()).removeChildQueue( queue.getQueueName()); cs.getCapacitySchedulerQueueManager().removeQueue(queue.getQueueName()); } }
private void validateQueueEntitlementChanges(AutoCreatedLeafQueue leafQueue, Map<String, QueueEntitlement> expectedQueueEntitlements, final List<QueueManagementChange> queueEntitlementChanges, Set<String> expectedNodeLabels) { boolean found = false; for (QueueManagementChange entitlementChange : queueEntitlementChanges) { if (leafQueue.getQueueName().equals( entitlementChange.getQueue().getQueueName())) { AutoCreatedLeafQueueConfig updatedQueueTemplate = entitlementChange.getUpdatedQueueTemplate(); for (String label : expectedNodeLabels) { QueueEntitlement newEntitlement = new QueueEntitlement( updatedQueueTemplate.getQueueCapacities().getCapacity(label), updatedQueueTemplate.getQueueCapacities().getMaximumCapacity (label)); assertEquals(expectedQueueEntitlements.get(label), newEntitlement); validateEffectiveMinResource(leafQueue, label, expectedQueueEntitlements); } found = true; break; } } if (!found) { fail( "Could not find the specified leaf queue in entitlement changes : " + leafQueue.getQueueName()); } }
TestCapacitySchedulerAutoCreatedQueueBase.setupQueueMappings(conf, "c", true, new int[] { 1, 2 }); TestCapacitySchedulerAutoCreatedQueueBase.setupQueueMappings(conf, "d", true, new int[] { 3, 4 }); TestCapacitySchedulerAutoCreatedQueueBase.setupQueueMappings(conf, "e", true, new int[] { 0 });
@Test public void testUpdatePlacementRulesFactory() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); // init queue mapping for UserGroupMappingRule and AppNameMappingRule setupQueueMappingsForRules(conf, PARENT_QUEUE, true, new int[] {1, 2, 3}); mockRM = new MockRM(conf); CapacityScheduler cs = (CapacityScheduler) mockRM.getResourceScheduler(); cs.updatePlacementRules(); mockRM.start(); cs.start(); List<PlacementRule> rules = cs.getRMContext() .getQueuePlacementManager().getPlacementRules(); List<String> placementRuleNames = new ArrayList<>(); for (PlacementRule pr : rules) { placementRuleNames.add(pr.getName()); } // verify both placement rules were added successfully assertThat(placementRuleNames, hasItems(QUEUE_MAPPING_RULE_USER_GROUP)); assertThat(placementRuleNames, hasItems(QUEUE_MAPPING_RULE_APP_NAME)); } }
@Before public void setUp() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); setupQueueMappings(conf, PARENT_QUEUE, true, new int[] { 0, 1, 2, 3 }); dispatcher = new SpyDispatcher(); rmAppEventEventHandler = new SpyDispatcher.SpyRMAppEventHandler(); dispatcher.register(RMAppEventType.class, rmAppEventEventHandler); RMNodeLabelsManager mgr = setupNodeLabelManager(conf); mockRM = new MockRM(conf) { protected RMNodeLabelsManager createNodeLabelManager() { return mgr; } }; cs = (CapacityScheduler) mockRM.getResourceScheduler(); cs.updatePlacementRules(); mockRM.start(); cs.start(); setupNodes(mockRM); }
public static CapacitySchedulerConfiguration setupQueueMappings( CapacitySchedulerConfiguration conf, String parentQueue, boolean overrideWithQueueMappings, int[] userIds) { List<String> queuePlacementRules = new ArrayList<>(); queuePlacementRules.add(YarnConfiguration.USER_GROUP_PLACEMENT_RULE); conf.setQueuePlacementRules(queuePlacementRules); List<UserGroupMappingPlacementRule.QueueMapping> existingMappings = conf.getQueueMappings(); //set queue mapping List<UserGroupMappingPlacementRule.QueueMapping> queueMappings = new ArrayList<>(); for (int i = 0; i < userIds.length; i++) { //Set C as parent queue name for auto queue creation UserGroupMappingPlacementRule.QueueMapping userQueueMapping = new UserGroupMappingPlacementRule.QueueMapping( UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, USER + userIds[i], getQueueMapping(parentQueue, USER + userIds[i])); queueMappings.add(userQueueMapping); } existingMappings.addAll(queueMappings); conf.setQueueMappings(existingMappings); //override with queue mappings conf.setOverrideWithQueueMappings(overrideWithQueueMappings); return conf; }
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, USER + sourceIds[i], getQueueMapping(parentQueue, USER + sourceIds[i])); queueMappingsForUG.add(userQueueMapping); getQueueMapping(parentQueue, USER + sourceIds[i])); queueMappingsForAN.add(queueMapping);
public static CapacitySchedulerConfiguration setupGroupQueueMappings (String parentQueue, CapacitySchedulerConfiguration conf, String leafQueueName) { List<UserGroupMappingPlacementRule.QueueMapping> existingMappings = conf.getQueueMappings(); //set queue mapping List<UserGroupMappingPlacementRule.QueueMapping> queueMappings = new ArrayList<>(); //setup group mapping conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, TestGroupsCaching.FakeunPrivilegedGroupMapping.class, ShellBasedUnixGroupsMapping.class); conf.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES, TEST_GROUPUSER +"=" + TEST_GROUP + ";invalid_user=invalid_group"); UserGroupMappingPlacementRule.QueueMapping userQueueMapping = new UserGroupMappingPlacementRule.QueueMapping( UserGroupMappingPlacementRule.QueueMapping.MappingType.GROUP, TEST_GROUP, getQueueMapping(parentQueue, leafQueueName)); queueMappings.add(userQueueMapping); existingMappings.addAll(queueMappings); conf.setQueueMappings(existingMappings); return conf; }