@Test public void testExplicitPropertyMappings() { Map<String, String> properties = new ImmutableMap.Builder<String, String>() .put("query.max-memory-per-node", "1GB") .put("query.max-total-memory-per-node", "3GB") .put("memory.heap-headroom-per-node", "1GB") .put("deprecated.legacy-system-pool-enabled", "true") .put("experimental.reserved-pool-enabled", "false") .build(); NodeMemoryConfig expected = new NodeMemoryConfig() .setLegacySystemPoolEnabled(true) .setMaxQueryMemoryPerNode(new DataSize(1, GIGABYTE)) .setMaxQueryTotalMemoryPerNode(new DataSize(3, GIGABYTE)) .setHeapHeadroom(new DataSize(1, GIGABYTE)) .setReservedPoolEnabled(false); assertFullMapping(properties, expected); }
private void configureMemoryPools(NodeMemoryConfig config, long availableMemory) { validateHeapHeadroom(config, availableMemory); maxMemory = new DataSize(availableMemory - config.getHeapHeadroom().toBytes(), BYTE); checkArgument( config.getMaxQueryMemoryPerNode().toBytes() <= config.getMaxQueryTotalMemoryPerNode().toBytes(), "Max query memory per node (%s) cannot be greater than the max query total memory per node (%s).", QUERY_MAX_MEMORY_PER_NODE_CONFIG, QUERY_MAX_TOTAL_MEMORY_PER_NODE_CONFIG); ImmutableMap.Builder<MemoryPoolId, MemoryPool> builder = ImmutableMap.builder(); long generalPoolSize = maxMemory.toBytes(); if (config.isReservedPoolEnabled()) { builder.put(RESERVED_POOL, new MemoryPool(RESERVED_POOL, config.getMaxQueryTotalMemoryPerNode())); generalPoolSize -= config.getMaxQueryTotalMemoryPerNode().toBytes(); } verify(generalPoolSize > 0, "general memory pool size is 0"); builder.put(GENERAL_POOL, new MemoryPool(GENERAL_POOL, new DataSize(generalPoolSize, BYTE))); this.pools = builder.build(); }
@Inject public LocalMemoryManager(NodeMemoryConfig config, ReservedSystemMemoryConfig systemMemoryConfig) { requireNonNull(config, "config is null"); long availableMemory = Runtime.getRuntime().maxMemory(); if (config.isLegacySystemPoolEnabled()) { configureLegacyMemoryPools(config, systemMemoryConfig, availableMemory); } else { configureMemoryPools(config, availableMemory); } }
@VisibleForTesting static void validateHeapHeadroom(NodeMemoryConfig config, long availableMemory) { long maxQueryTotalMemoryPerNode = config.getMaxQueryTotalMemoryPerNode().toBytes(); long heapHeadroom = config.getHeapHeadroom().toBytes(); // (availableMemory - maxQueryTotalMemoryPerNode) bytes will be available for the general pool and the // headroom/untracked allocations, so the heapHeadroom cannot be larger than that space. if (heapHeadroom < 0 || heapHeadroom + maxQueryTotalMemoryPerNode > availableMemory) { throw new IllegalArgumentException( format("Invalid memory configuration. The sum of max total query memory per node (%s) and heap headroom (%s) cannot be larger than the available heap memory (%s)", maxQueryTotalMemoryPerNode, heapHeadroom, availableMemory)); } }
private void configureLegacyMemoryPools(NodeMemoryConfig config, ReservedSystemMemoryConfig systemMemoryConfig, long availableMemory) { checkArgument(systemMemoryConfig.getReservedSystemMemory().toBytes() < availableMemory, "Reserved memory %s is greater than available heap %s", systemMemoryConfig.getReservedSystemMemory(), new DataSize(availableMemory, BYTE)); maxMemory = new DataSize(availableMemory - systemMemoryConfig.getReservedSystemMemory().toBytes(), BYTE); ImmutableMap.Builder<MemoryPoolId, MemoryPool> builder = ImmutableMap.builder(); checkArgument(config.getMaxQueryMemoryPerNode().toBytes() <= maxMemory.toBytes(), format("%s set to %s, but only %s of useable heap available", QUERY_MAX_MEMORY_PER_NODE_CONFIG, config.getMaxQueryMemoryPerNode(), maxMemory)); long generalPoolSize = maxMemory.toBytes(); if (config.isReservedPoolEnabled()) { builder.put(RESERVED_POOL, new MemoryPool(RESERVED_POOL, config.getMaxQueryMemoryPerNode())); generalPoolSize -= config.getMaxQueryMemoryPerNode().toBytes(); } builder.put(GENERAL_POOL, new MemoryPool(GENERAL_POOL, new DataSize(generalPoolSize, BYTE))); builder.put(SYSTEM_POOL, new MemoryPool(SYSTEM_POOL, systemMemoryConfig.getReservedSystemMemory())); this.pools = builder.build(); }
@Test(expectedExceptions = IllegalArgumentException.class) public void testInvalidValues() { NodeMemoryConfig config = new NodeMemoryConfig(); config.setMaxQueryTotalMemoryPerNode(new DataSize(1, GIGABYTE)); config.setHeapHeadroom(new DataSize(3.1, GIGABYTE)); // In this case we have 4GB - 1GB = 3GB available memory for the general pool // and the heap headroom and the config is more than that. validateHeapHeadroom(config, new DataSize(4, GIGABYTE).toBytes()); } }
this.enabled = serverConfig.isCoordinator(); this.killOnOutOfMemoryDelay = config.getKillOnOutOfMemoryDelay(); this.isLegacySystemPoolEnabled = nodeMemoryConfig.isLegacySystemPoolEnabled(); this.isWorkScheduledOnCoordinator = schedulerConfig.isIncludeCoordinator(); "maxQueryMemory cannot be greater than maxQueryTotalMemory"); this.pools = createClusterMemoryPools(nodeMemoryConfig.isLegacySystemPoolEnabled(), nodeMemoryConfig.isReservedPoolEnabled());
DataSize maxQueryUserMemoryPerNode = nodeMemoryConfig.getMaxQueryMemoryPerNode(); DataSize maxQueryTotalMemoryPerNode = nodeMemoryConfig.getMaxQueryTotalMemoryPerNode(); DataSize maxQuerySpillPerNode = nodeSpillConfig.getQueryMaxSpillPerNode();
public TestSqlTaskManager() { localMemoryManager = new LocalMemoryManager(new NodeMemoryConfig(), new ReservedSystemMemoryConfig()); localSpillManager = new LocalSpillManager(new NodeSpillConfig()); taskExecutor = new TaskExecutor(8, 16, 3, 4, Ticker.systemTicker()); taskExecutor.start(); taskManagementExecutor = new TaskManagementExecutor(); }
@Test public void testDefaults() { ConfigAssertions.assertRecordedDefaults(ConfigAssertions.recordDefaults(NodeMemoryConfig.class) .setLegacySystemPoolEnabled(false) .setMaxQueryMemoryPerNode(new DataSize(AVAILABLE_HEAP_MEMORY * 0.1, BYTE)) .setMaxQueryTotalMemoryPerNode(new DataSize(AVAILABLE_HEAP_MEMORY * 0.3, BYTE)) .setHeapHeadroom(new DataSize(AVAILABLE_HEAP_MEMORY * 0.3, BYTE)) .setReservedPoolEnabled(true)); }
public SqlTaskManager createSqlTaskManager(TaskManagerConfig config) { return new SqlTaskManager( createTestingPlanner(), new MockLocationFactory(), taskExecutor, createTestSplitMonitor(), new NodeInfo("test"), localMemoryManager, taskManagementExecutor, config, new NodeMemoryConfig(), localSpillManager, new NodeSpillConfig(), new TestingGcMonitor()); }
DataSize maxQuerySpillPerNode) if (nodeMemoryConfig.isLegacySystemPoolEnabled()) { Optional<MemoryPool> systemPool = localMemoryManager.getSystemPool(); verify(systemPool.isPresent(), "systemPool must be present");