public ParentQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { super(cs, queueName, parent, old); this.scheduler = cs; this.rootQueue = (parent == null); float rawCapacity = cs.getConfiguration().getNonLabeledQueueCapacity(getQueuePath()); if (rootQueue && (rawCapacity != CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE)) { throw new IllegalArgumentException("Illegal " + "capacity of " + rawCapacity + " for queue " + queueName + ". Must be " + CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE); } this.childQueues = new ArrayList<>(); setupQueueConfigs(cs.getClusterResource()); LOG.info("Initialized parent-queue " + queueName + " name=" + queueName + ", fullname=" + getQueuePath()); }
public ParentQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { super(cs, queueName, parent, old); this.scheduler = cs; this.queueComparator = cs.getQueueComparator(); this.rootQueue = (parent == null); float rawCapacity = cs.getConfiguration().getNonLabeledQueueCapacity(getQueuePath()); if (rootQueue && (rawCapacity != CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE)) { throw new IllegalArgumentException("Illegal " + "capacity of " + rawCapacity + " for queue " + queueName + ". Must be " + CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE); } this.childQueues = new TreeSet<CSQueue>(queueComparator); setupQueueConfigs(cs.getClusterResource()); LOG.info("Initialized parent-queue " + queueName + " name=" + queueName + ", fullname=" + getQueuePath()); }
public ParentQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { super(cs, queueName, parent, old); this.scheduler = cs; this.queueComparator = cs.getQueueComparator(); this.rootQueue = (parent == null); float rawCapacity = cs.getConfiguration().getNonLabeledQueueCapacity(getQueuePath()); if (rootQueue && (rawCapacity != CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE)) { throw new IllegalArgumentException("Illegal " + "capacity of " + rawCapacity + " for queue " + queueName + ". Must be " + CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE); } this.childQueues = new TreeSet<CSQueue>(queueComparator); setupQueueConfigs(cs.getClusterResource()); LOG.info("Initialized parent-queue " + queueName + " name=" + queueName + ", fullname=" + getQueuePath()); }
private static void loadCapacitiesByLabelsFromConf(String queuePath, QueueCapacities queueCapacities, CapacitySchedulerConfiguration csConf) { queueCapacities.clearConfigurableFields(); Set<String> configuredNodelabels = csConf.getConfiguredNodeLabels(queuePath); for (String label : configuredNodelabels) { if (label.equals(CommonNodeLabelsManager.NO_LABEL)) { queueCapacities.setCapacity(CommonNodeLabelsManager.NO_LABEL, csConf.getNonLabeledQueueCapacity(queuePath) / 100); queueCapacities.setMaximumCapacity(CommonNodeLabelsManager.NO_LABEL, csConf.getNonLabeledQueueMaximumCapacity(queuePath) / 100); } else { queueCapacities.setCapacity(label, csConf.getLabeledQueueCapacity(queuePath, label) / 100); queueCapacities.setMaximumCapacity(label, csConf.getLabeledQueueMaximumCapacity(queuePath, label) / 100); } } }
private static void loadCapacitiesByLabelsFromConf(String queuePath, QueueCapacities queueCapacities, CapacitySchedulerConfiguration csConf) { queueCapacities.clearConfigurableFields(); Set<String> configuredNodelabels = csConf.getConfiguredNodeLabels(queuePath); for (String label : configuredNodelabels) { if (label.equals(CommonNodeLabelsManager.NO_LABEL)) { queueCapacities.setCapacity(CommonNodeLabelsManager.NO_LABEL, csConf.getNonLabeledQueueCapacity(queuePath) / 100); queueCapacities.setMaximumCapacity(CommonNodeLabelsManager.NO_LABEL, csConf.getNonLabeledQueueMaximumCapacity(queuePath) / 100); } else { queueCapacities.setCapacity(label, csConf.getLabeledQueueCapacity(queuePath, label) / 100); queueCapacities.setMaximumCapacity(label, csConf.getLabeledQueueMaximumCapacity(queuePath, label) / 100); } } }
private static void loadCapacitiesByLabelsFromConf(String queuePath, QueueCapacities queueCapacities, CapacitySchedulerConfiguration csConf) { queueCapacities.clearConfigurableFields(); Set<String> configuredNodelabels = csConf.getConfiguredNodeLabels(queuePath); for (String label : configuredNodelabels) { if (label.equals(CommonNodeLabelsManager.NO_LABEL)) { queueCapacities.setCapacity(label, csConf.getNonLabeledQueueCapacity(queuePath) / 100); queueCapacities.setMaximumCapacity(label, csConf.getNonLabeledQueueMaximumCapacity(queuePath) / 100); queueCapacities.setMaxAMResourcePercentage( label, csConf.getMaximumAMResourcePercentPerPartition(queuePath, label)); } else { queueCapacities.setCapacity(label, csConf.getLabeledQueueCapacity(queuePath, label) / 100); queueCapacities.setMaximumCapacity(label, csConf.getLabeledQueueMaximumCapacity(queuePath, label) / 100); queueCapacities.setMaxAMResourcePercentage(label, csConf.getMaximumAMResourcePercentPerPartition(queuePath, label)); } } }
@Test public void testUpdateQueueCapacity() throws Exception { WebResource r = resource(); ClientResponse response; // Update root.a and root.b capacity to 50. SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); Map<String, String> updateParam = new HashMap<>(); updateParam.put(CapacitySchedulerConfiguration.CAPACITY, "50"); QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam); QueueConfigInfo bUpdateInfo = new QueueConfigInfo("root.b", updateParam); updateInfo.getUpdateQueueInfo().add(aUpdateInfo); updateInfo.getUpdateQueueInfo().add(bUpdateInfo); response = r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.a"), 0.01f); assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f); }
@Test public void testAddWithUpdate() throws Exception { WebResource r = resource(); ClientResponse response; // Add root.d with capacity 25, reducing root.b capacity from 75 to 50. SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); Map<String, String> dCapacity = new HashMap<>(); dCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "25"); Map<String, String> bCapacity = new HashMap<>(); bCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "50"); QueueConfigInfo d = new QueueConfigInfo("root.d", dCapacity); QueueConfigInfo b = new QueueConfigInfo("root.b", bCapacity); updateInfo.getAddQueueInfo().add(d); updateInfo.getUpdateQueueInfo().add(b); response = r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); assertEquals(4, newCSConf.getQueues("root").length); assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity("root.d"), 0.01f); assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f); }
private void verifySubQueueGeneric(String q, QueueInfo info, float parentAbsCapacity, float parentAbsMaxCapacity) throws Exception { String[] qArr = q.split("\\."); assertTrue("q name invalid: " + q, qArr.length > 1); String qshortName = qArr[qArr.length - 1]; assertEquals("usedCapacity doesn't match", 0, info.usedCapacity, 1e-3f); assertEquals("capacity doesn't match", csConf.getNonLabeledQueueCapacity(q), info.capacity, 1e-3f); float expectCapacity = csConf.getNonLabeledQueueMaximumCapacity(q); float expectAbsMaxCapacity = parentAbsMaxCapacity * (info.maxCapacity/100); if (CapacitySchedulerConfiguration.UNDEFINED == expectCapacity) { expectCapacity = 100; expectAbsMaxCapacity = 100; } assertEquals("maxCapacity doesn't match", expectCapacity, info.maxCapacity, 1e-3f); assertEquals("absoluteCapacity doesn't match", parentAbsCapacity * (info.capacity/100), info.absoluteCapacity, 1e-3f); assertEquals("absoluteMaxCapacity doesn't match", expectAbsMaxCapacity, info.absoluteMaxCapacity, 1e-3f); assertEquals("absoluteUsedCapacity doesn't match", 0, info.absoluteUsedCapacity, 1e-3f); assertEquals("numApplications doesn't match", 0, info.numApplications); assertTrue("queueName doesn't match, got: " + info.queueName + " expected: " + q, qshortName.matches(info.queueName)); assertTrue("state doesn't match", (csConf.getState(q).toString()).matches(info.state)); }
assertEquals(4, newCSConf.getQueues("root").length); assertEquals(2, newCSConf.getQueues("root.d").length); assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d1"), 0.01f); assertEquals(75.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d2"), 0.01f);
private void verifySubQueueGeneric(String q, QueueInfo info, float parentAbsCapacity, float parentAbsMaxCapacity) throws Exception { String[] qArr = q.split("\\."); assertTrue("q name invalid: " + q, qArr.length > 1); String qshortName = qArr[qArr.length - 1]; assertEquals("usedCapacity doesn't match", 0, info.usedCapacity, 1e-3f); assertEquals("capacity doesn't match", csConf.getNonLabeledQueueCapacity(q), info.capacity, 1e-3f); float expectCapacity = csConf.getNonLabeledQueueMaximumCapacity(q); float expectAbsMaxCapacity = parentAbsMaxCapacity * (info.maxCapacity/100); if (CapacitySchedulerConfiguration.UNDEFINED == expectCapacity) { expectCapacity = 100; expectAbsMaxCapacity = 100; } assertEquals("maxCapacity doesn't match", expectCapacity, info.maxCapacity, 1e-3f); assertEquals("absoluteCapacity doesn't match", parentAbsCapacity * (info.capacity/100), info.absoluteCapacity, 1e-3f); assertEquals("absoluteMaxCapacity doesn't match", expectAbsMaxCapacity, info.absoluteMaxCapacity, 1e-3f); assertEquals("absoluteUsedCapacity doesn't match", 0, info.absoluteUsedCapacity, 1e-3f); assertEquals("numApplications doesn't match", 0, info.numApplications); assertTrue("queueName doesn't match, got: " + info.queueName + " expected: " + q, qshortName.matches(info.queueName)); assertTrue("state doesn't match", (csConf.getState(q).toString()).matches(info.state)); }
@Test public void testRemoveParentQueueWithCapacity() throws Exception { WebResource r = resource(); ClientResponse response; stopQueue("root.a", "root.a.a1", "root.a.a2"); // Remove root.a (parent queue) with capacity 25 SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); updateInfo.getRemoveQueueInfo().add("root.a"); // Set root.b capacity to 100 Map<String, String> bCapacity = new HashMap<>(); bCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "100"); QueueConfigInfo b = new QueueConfigInfo("root.b", bCapacity); updateInfo.getUpdateQueueInfo().add(b); response = r.path("ws").path("v1").path("cluster") .path("scheduler-conf").queryParam("user.name", userName) .accept(MediaType.APPLICATION_JSON) .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON) .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); assertEquals(2, newCSConf.getQueues("root").length); assertEquals(100.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f); }