private static void assertGroupInfoEquals(ResourceGroupInfo actual, ResourceGroupInfo expected) { assertTrue(actual.getSchedulingWeight() == expected.getSchedulingWeight() && actual.getSoftConcurrencyLimit() == expected.getSoftConcurrencyLimit() && actual.getHardConcurrencyLimit() == expected.getHardConcurrencyLimit() && actual.getMaxQueuedQueries() == expected.getMaxQueuedQueries() && actual.getNumQueuedQueries() == expected.getNumQueuedQueries() && actual.getNumRunningQueries() == expected.getNumRunningQueries() && actual.getNumEligibleSubGroups() == expected.getNumEligibleSubGroups() && Objects.equals(actual.getId(), expected.getId()) && actual.getState() == expected.getState() && actual.getSchedulingPolicy() == expected.getSchedulingPolicy() && Objects.equals(actual.getSoftMemoryLimit(), expected.getSoftMemoryLimit()) && Objects.equals(actual.getMemoryUsage(), expected.getMemoryUsage())); } }
public ResourceGroupInfo getInfo() { synchronized (root) { return new ResourceGroupInfo( id, getState(), schedulingPolicy, schedulingWeight, DataSize.succinctBytes(softMemoryLimitBytes), softConcurrencyLimit, hardConcurrencyLimit, maxQueuedQueries, DataSize.succinctBytes(cachedMemoryUsageBytes), getQueuedQueries(), getRunningQueries(), eligibleSubGroups.size(), subGroups.values().stream() .filter(group -> group.getRunningQueries() + group.getQueuedQueries() > 0) .map(InternalResourceGroup::getSummaryInfo) .collect(toImmutableList()), null); } }
assertEquals(rootInfo.getId(), root.getId()); assertEquals(rootInfo.getState(), CAN_RUN); assertEquals(rootInfo.getSoftMemoryLimit(), root.getSoftMemoryLimit()); assertEquals(rootInfo.getMemoryUsage(), new DataSize(0, BYTE)); assertEquals(rootInfo.getSubGroups().size(), 2); assertGroupInfoEquals(rootInfo.getSubGroups().get(0), rootA.getInfo()); assertEquals(rootInfo.getSubGroups().get(0).getId(), rootA.getId()); assertEquals(rootInfo.getSubGroups().get(0).getState(), CAN_QUEUE); assertEquals(rootInfo.getSubGroups().get(0).getSoftMemoryLimit(), rootA.getSoftMemoryLimit()); assertEquals(rootInfo.getSubGroups().get(0).getHardConcurrencyLimit(), rootA.getHardConcurrencyLimit()); assertEquals(rootInfo.getSubGroups().get(0).getMaxQueuedQueries(), rootA.getMaxQueuedQueries()); assertEquals(rootInfo.getSubGroups().get(0).getNumEligibleSubGroups(), 2); assertEquals(rootInfo.getSubGroups().get(0).getNumRunningQueries(), 0); assertEquals(rootInfo.getSubGroups().get(0).getNumQueuedQueries(), 10); assertGroupInfoEquals(rootInfo.getSubGroups().get(1), rootB.getInfo()); assertEquals(rootInfo.getSubGroups().get(1).getId(), rootB.getId()); assertEquals(rootInfo.getSubGroups().get(1).getState(), CAN_QUEUE); assertEquals(rootInfo.getSubGroups().get(1).getSoftMemoryLimit(), rootB.getSoftMemoryLimit()); assertEquals(rootInfo.getSubGroups().get(1).getHardConcurrencyLimit(), rootB.getHardConcurrencyLimit()); assertEquals(rootInfo.getSubGroups().get(1).getMaxQueuedQueries(), rootB.getMaxQueuedQueries()); assertEquals(rootInfo.getSubGroups().get(1).getNumEligibleSubGroups(), 0); assertEquals(rootInfo.getSubGroups().get(1).getNumRunningQueries(), 1); assertEquals(rootInfo.getSubGroups().get(1).getNumQueuedQueries(), 9); assertEquals(rootInfo.getSoftConcurrencyLimit(), root.getSoftConcurrencyLimit()); assertEquals(rootInfo.getHardConcurrencyLimit(), root.getHardConcurrencyLimit()); assertEquals(rootInfo.getMaxQueuedQueries(), root.getMaxQueuedQueries()); assertEquals(rootInfo.getNumQueuedQueries(), 19); assertEquals(rootInfo.getRunningQueries().size(), 1); QueryStateInfo queryInfo = rootInfo.getRunningQueries().get(0);
@Test public void testPathToRoot() throws Exception { try (DistributedQueryRunner queryRunner = TpchQueryRunnerBuilder.builder().build()) { queryRunner.installPlugin(new ResourceGroupManagerPlugin()); InternalResourceGroupManager<?> manager = getResourceGroupManager(queryRunner); manager.setConfigurationManager("file", ImmutableMap.of( "resource-groups.config-file", getResourceFilePath("resource_groups_config_dashboard.json"))); queryRunner.execute(testSessionBuilder().setCatalog("tpch").setSchema("tiny").setSource("dashboard-foo").build(), "SELECT COUNT(*), clerk FROM orders GROUP BY clerk"); List<ResourceGroupInfo> path = manager.getPathToRoot(new ResourceGroupId(new ResourceGroupId(new ResourceGroupId("global"), "user-user"), "dashboard-user")); assertEquals(path.size(), 3); assertTrue(path.get(1).getSubGroups() != null); assertEquals(path.get(2).getId(), new ResourceGroupId("global")); assertEquals(path.get(2).getHardConcurrencyLimit(), 100); assertEquals(path.get(2).getRunningQueries(), null); } }
assertEquals(rootAInfo.getId(), expectedRootAInfo.getId()); assertEquals(rootAInfo.getState(), expectedRootAInfo.getState()); assertEquals(rootAInfo.getNumRunningQueries(), expectedRootAInfo.getNumRunningQueries()); assertEquals(rootAInfo.getNumQueuedQueries(), expectedRootAInfo.getNumQueuedQueries()); assertEquals(actualRootInfo.getId(), expectedRootInfo.getId()); assertEquals(actualRootInfo.getState(), expectedRootInfo.getState()); assertEquals(actualRootInfo.getNumRunningQueries(), expectedRootInfo.getNumRunningQueries()); assertEquals(actualRootInfo.getNumQueuedQueries(), expectedRootInfo.getNumQueuedQueries());
@Test(timeOut = 60_000) public void testRunningQuery() throws Exception { queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk"); while (true) { ResourceGroupInfo global = queryRunner.getCoordinator().getResourceGroupManager().get().getResourceGroupInfo(new ResourceGroupId(new ResourceGroupId("global"), "bi-user")); if (global.getSoftMemoryLimit().toBytes() > 0) { break; } TimeUnit.SECONDS.sleep(2); } }
assertEquals(info.getNumRunningQueries(), 0); assertEquals(info.getNumQueuedQueries(), 40); root.processQueuedQueries(); info = root.getInfo(); assertEquals(info.getNumRunningQueries(), 4); assertEquals(info.getNumQueuedQueries(), 36); assertEquals(info.getNumRunningQueries(), 4); assertEquals(info.getNumQueuedQueries(), 32); root.processQueuedQueries(); info = root.getInfo(); assertEquals(info.getNumRunningQueries(), 4); assertEquals(info.getNumQueuedQueries(), 32); root.processQueuedQueries(); info = root.getInfo(); assertEquals(info.getNumRunningQueries(), 10); assertEquals(info.getNumQueuedQueries(), 26);
assertEquals(query2a.getState(), QUEUED); assertEquals(root.getInfo().getNumEligibleSubGroups(), 2); assertEquals(root.getOrCreateSubGroup("1").getQueuedQueries(), 2); assertEquals(root.getOrCreateSubGroup("2").getQueuedQueries(), 1); assertEquals(root.getSchedulingPolicy(), FAIR); root.setSchedulingPolicy(QUERY_PRIORITY); assertEquals(root.getInfo().getNumEligibleSubGroups(), 2); assertEquals(root.getOrCreateSubGroup("1").getQueuedQueries(), 2); assertEquals(root.getOrCreateSubGroup("2").getQueuedQueries(), 1);
assertEquals(rootInfo.getId(), root.getId()); assertEquals(rootInfo.getState(), CAN_RUN); assertEquals(rootInfo.getSoftMemoryLimit(), root.getSoftMemoryLimit()); assertEquals(rootInfo.getMemoryUsage(), new DataSize(0, BYTE)); assertEquals(rootInfo.getSubGroups().size(), 2); assertGroupInfoEquals(rootInfo.getSubGroups().get(0), rootA.getInfo()); assertEquals(rootInfo.getSubGroups().get(0).getId(), rootA.getId()); assertEquals(rootInfo.getSubGroups().get(0).getState(), CAN_QUEUE); assertEquals(rootInfo.getSubGroups().get(0).getSoftMemoryLimit(), rootA.getSoftMemoryLimit()); assertEquals(rootInfo.getSubGroups().get(0).getHardConcurrencyLimit(), rootA.getHardConcurrencyLimit()); assertEquals(rootInfo.getSubGroups().get(0).getMaxQueuedQueries(), rootA.getMaxQueuedQueries()); assertEquals(rootInfo.getSubGroups().get(0).getNumEligibleSubGroups(), 2); assertEquals(rootInfo.getSubGroups().get(0).getNumRunningQueries(), 0); assertEquals(rootInfo.getSubGroups().get(0).getNumQueuedQueries(), 10); assertGroupInfoEquals(rootInfo.getSubGroups().get(1), rootB.getInfo()); assertEquals(rootInfo.getSubGroups().get(1).getId(), rootB.getId()); assertEquals(rootInfo.getSubGroups().get(1).getState(), CAN_QUEUE); assertEquals(rootInfo.getSubGroups().get(1).getSoftMemoryLimit(), rootB.getSoftMemoryLimit()); assertEquals(rootInfo.getSubGroups().get(1).getHardConcurrencyLimit(), rootB.getHardConcurrencyLimit()); assertEquals(rootInfo.getSubGroups().get(1).getMaxQueuedQueries(), rootB.getMaxQueuedQueries()); assertEquals(rootInfo.getSubGroups().get(1).getNumEligibleSubGroups(), 0); assertEquals(rootInfo.getSubGroups().get(1).getNumRunningQueries(), 1); assertEquals(rootInfo.getSubGroups().get(1).getNumQueuedQueries(), 9); assertEquals(rootInfo.getSoftConcurrencyLimit(), root.getSoftConcurrencyLimit()); assertEquals(rootInfo.getHardConcurrencyLimit(), root.getHardConcurrencyLimit()); assertEquals(rootInfo.getMaxQueuedQueries(), root.getMaxQueuedQueries()); assertEquals(rootInfo.getNumQueuedQueries(), 19); assertEquals(rootInfo.getRunningQueries().size(), 1); QueryStateInfo queryInfo = rootInfo.getRunningQueries().get(0);
assertEquals(rootAInfo.getId(), expectedRootAInfo.getId()); assertEquals(rootAInfo.getState(), expectedRootAInfo.getState()); assertEquals(rootAInfo.getNumRunningQueries(), expectedRootAInfo.getNumRunningQueries()); assertEquals(rootAInfo.getNumQueuedQueries(), expectedRootAInfo.getNumQueuedQueries()); assertEquals(actualRootInfo.getId(), expectedRootInfo.getId()); assertEquals(actualRootInfo.getState(), expectedRootInfo.getState()); assertEquals(actualRootInfo.getNumRunningQueries(), expectedRootInfo.getNumRunningQueries()); assertEquals(actualRootInfo.getNumQueuedQueries(), expectedRootInfo.getNumQueuedQueries());
public static void waitForGlobalResourceGroup(DistributedQueryRunner queryRunner) throws InterruptedException { long startTime = System.nanoTime(); while (true) { SECONDS.sleep(1); ResourceGroupInfo global = getResourceGroupManager(queryRunner).getResourceGroupInfo(new ResourceGroupId("global")); if (global.getSoftMemoryLimit().toBytes() > 0) { break; } assertLessThan(nanosSince(startTime).roundTo(SECONDS), 60L); } }
assertEquals(info.getNumRunningQueries(), 0); assertEquals(info.getNumQueuedQueries(), 40); root.processQueuedQueries(); info = root.getInfo(); assertEquals(info.getNumRunningQueries(), 4); assertEquals(info.getNumQueuedQueries(), 36); assertEquals(info.getNumRunningQueries(), 4); assertEquals(info.getNumQueuedQueries(), 32); root.processQueuedQueries(); info = root.getInfo(); assertEquals(info.getNumRunningQueries(), 4); assertEquals(info.getNumQueuedQueries(), 32); root.processQueuedQueries(); info = root.getInfo(); assertEquals(info.getNumRunningQueries(), 10); assertEquals(info.getNumQueuedQueries(), 26);
assertEquals(query2a.getState(), QUEUED); assertEquals(root.getInfo().getNumEligibleSubGroups(), 2); assertEquals(root.getOrCreateSubGroup("1").getQueuedQueries(), 2); assertEquals(root.getOrCreateSubGroup("2").getQueuedQueries(), 1); assertEquals(root.getSchedulingPolicy(), FAIR); root.setSchedulingPolicy(QUERY_PRIORITY); assertEquals(root.getInfo().getNumEligibleSubGroups(), 2); assertEquals(root.getOrCreateSubGroup("1").getQueuedQueries(), 2); assertEquals(root.getOrCreateSubGroup("2").getQueuedQueries(), 1);
private static void assertGroupInfoEquals(ResourceGroupInfo actual, ResourceGroupInfo expected) { assertTrue(actual.getSchedulingWeight() == expected.getSchedulingWeight() && actual.getSoftConcurrencyLimit() == expected.getSoftConcurrencyLimit() && actual.getHardConcurrencyLimit() == expected.getHardConcurrencyLimit() && actual.getMaxQueuedQueries() == expected.getMaxQueuedQueries() && actual.getNumQueuedQueries() == expected.getNumQueuedQueries() && actual.getNumRunningQueries() == expected.getNumRunningQueries() && actual.getNumEligibleSubGroups() == expected.getNumEligibleSubGroups() && Objects.equals(actual.getId(), expected.getId()) && actual.getState() == expected.getState() && actual.getSchedulingPolicy() == expected.getSchedulingPolicy() && Objects.equals(actual.getSoftMemoryLimit(), expected.getSoftMemoryLimit()) && Objects.equals(actual.getMemoryUsage(), expected.getMemoryUsage())); } }
public ResourceGroupInfo getInfo() { synchronized (root) { return new ResourceGroupInfo( id, getState(), schedulingPolicy, schedulingWeight, DataSize.succinctBytes(softMemoryLimitBytes), softConcurrencyLimit, hardConcurrencyLimit, maxQueuedQueries, DataSize.succinctBytes(cachedMemoryUsageBytes), getQueuedQueries(), getRunningQueries(), eligibleSubGroups.size(), subGroups.values().stream() .filter(group -> group.getRunningQueries() + group.getQueuedQueries() > 0) .map(InternalResourceGroup::getSummaryInfo) .collect(toImmutableList()), null); } }
private ResourceGroupInfo getSummaryInfo() { synchronized (root) { return new ResourceGroupInfo( id, getState(), schedulingPolicy, schedulingWeight, DataSize.succinctBytes(softMemoryLimitBytes), softConcurrencyLimit, hardConcurrencyLimit, maxQueuedQueries, DataSize.succinctBytes(cachedMemoryUsageBytes), getQueuedQueries(), getRunningQueries(), eligibleSubGroups.size(), null, null); } }
private ResourceGroupInfo getSummaryInfo() { synchronized (root) { return new ResourceGroupInfo( id, getState(), schedulingPolicy, schedulingWeight, DataSize.succinctBytes(softMemoryLimitBytes), softConcurrencyLimit, hardConcurrencyLimit, maxQueuedQueries, DataSize.succinctBytes(cachedMemoryUsageBytes), getQueuedQueries(), getRunningQueries(), eligibleSubGroups.size(), null, null); } }