public synchronized MemoryPoolInfo getInfo() { return new MemoryPoolInfo( totalDistributedBytes, reservedDistributedBytes, reservedRevocableDistributedBytes, ImmutableMap.copyOf(queryMemoryReservations), ImmutableMap.copyOf(queryMemoryAllocations), ImmutableMap.copyOf(queryMemoryRevocableReservations)); }
if (poolInfo != null) { nodes++; if (poolInfo.getFreeBytes() + poolInfo.getReservedRevocableBytes() <= 0) { blockedNodes++; totalDistributedBytes += poolInfo.getMaxBytes(); reservedDistributedBytes += poolInfo.getReservedBytes(); reservedRevocableDistributedBytes += poolInfo.getReservedRevocableBytes(); for (Map.Entry<QueryId, Long> entry : poolInfo.getQueryMemoryReservations().entrySet()) { queryMemoryReservations.merge(entry.getKey(), entry.getValue(), Long::sum); for (Map.Entry<QueryId, List<MemoryAllocation>> entry : poolInfo.getQueryMemoryAllocations().entrySet()) { queryMemoryAllocations.merge(entry.getKey(), entry.getValue(), this::mergeQueryAllocations); for (Map.Entry<QueryId, Long> entry : poolInfo.getQueryMemoryRevocableReservations().entrySet()) { queryMemoryRevocableReservations.merge(entry.getKey(), entry.getValue(), Long::sum);
@Override public Optional<QueryId> chooseQueryToKill(List<QueryMemoryInfo> runningQueries, List<MemoryInfo> nodes) { Map<QueryId, Long> memoryReservationOnBlockedNodes = new HashMap<>(); for (MemoryInfo node : nodes) { MemoryPoolInfo generalPool = node.getPools().get(GENERAL_POOL); if (generalPool == null) { continue; } if (generalPool.getFreeBytes() + generalPool.getReservedRevocableBytes() > 0) { continue; } Map<QueryId, Long> queryMemoryReservations = generalPool.getQueryMemoryReservations(); queryMemoryReservations.forEach((queryId, memoryReservation) -> { memoryReservationOnBlockedNodes.compute(queryId, (id, oldValue) -> oldValue == null ? memoryReservation : oldValue + memoryReservation); }); } return memoryReservationOnBlockedNodes.entrySet().stream() .max(comparingLong(Map.Entry::getValue)) .map(Map.Entry::getKey); } }
private void logQueryKill(QueryId killedQueryId, List<MemoryInfo> nodes) { if (!log.isInfoEnabled()) { return; } StringBuilder nodeDescription = new StringBuilder(); nodeDescription.append("Query Kill Decision: Killed ").append(killedQueryId).append("\n"); for (MemoryInfo node : nodes) { MemoryPoolInfo memoryPoolInfo = node.getPools().get(GENERAL_POOL); if (memoryPoolInfo == null) { continue; } nodeDescription.append("Query Kill Scenario: "); nodeDescription.append("MaxBytes ").append(memoryPoolInfo.getMaxBytes()).append(' '); nodeDescription.append("FreeBytes ").append(memoryPoolInfo.getFreeBytes() + memoryPoolInfo.getReservedRevocableBytes()).append(' '); nodeDescription.append("Queries "); Joiner.on(",").withKeyValueSeparator("=").appendTo(nodeDescription, memoryPoolInfo.getQueryMemoryReservations()); nodeDescription.append('\n'); } log.info(nodeDescription.toString()); }
protected AbstractResourceConfigurationManager(ClusterMemoryPoolManager memoryPoolManager) { memoryPoolManager.addChangeListener(new MemoryPoolId("general"), poolInfo -> { Map<ResourceGroup, DataSize> memoryLimits = new HashMap<>(); synchronized (generalPoolMemoryFraction) { for (Map.Entry<ResourceGroup, Double> entry : generalPoolMemoryFraction.entrySet()) { double bytes = poolInfo.getMaxBytes() * entry.getValue(); // setSoftMemoryLimit() acquires a lock on the root group of its tree, which could cause a deadlock if done while holding the "generalPoolMemoryFraction" lock memoryLimits.put(entry.getKey(), new DataSize(bytes, BYTE)); } generalPoolBytes = poolInfo.getMaxBytes(); } for (Map.Entry<ResourceGroup, DataSize> entry : memoryLimits.entrySet()) { entry.getKey().setSoftMemoryLimit(entry.getValue()); } }); }
protected AbstractResourceConfigurationManager(ClusterMemoryPoolManager memoryPoolManager) { memoryPoolManager.addChangeListener(new MemoryPoolId("general"), poolInfo -> { Map<ResourceGroup, DataSize> memoryLimits = new HashMap<>(); synchronized (generalPoolMemoryFraction) { for (Map.Entry<ResourceGroup, Double> entry : generalPoolMemoryFraction.entrySet()) { double bytes = poolInfo.getMaxBytes() * entry.getValue(); // setSoftMemoryLimit() acquires a lock on the root group of its tree, which could cause a deadlock if done while holding the "generalPoolMemoryFraction" lock memoryLimits.put(entry.getKey(), new DataSize(bytes, BYTE)); } generalPoolBytes = poolInfo.getMaxBytes(); } for (Map.Entry<ResourceGroup, DataSize> entry : memoryLimits.entrySet()) { entry.getKey().setSoftMemoryLimit(entry.getValue()); } }); }
public synchronized MemoryPoolInfo getInfo() { Map<QueryId, List<MemoryAllocation>> memoryAllocations = new HashMap<>(); for (Entry<QueryId, Map<String, Long>> entry : taggedMemoryAllocations.entrySet()) { List<MemoryAllocation> allocations = new ArrayList<>(); if (entry.getValue() != null) { entry.getValue().forEach((tag, allocation) -> allocations.add(new MemoryAllocation(tag, allocation))); } memoryAllocations.put(entry.getKey(), allocations); } return new MemoryPoolInfo(maxBytes, reservedBytes, reservedRevocableBytes, queryMemoryReservations, memoryAllocations, queryMemoryRevocableReservations); }
pools.put( GENERAL_POOL, new MemoryPoolInfo( maxGeneralPoolBytes, nodeReservation.getGeneral().getTotalReservedBytes(), pools.put( RESERVED_POOL, new MemoryPoolInfo( maxReservedPoolBytes, nodeReservation.getReserved().getTotalReservedBytes(),