@Override public synchronized String toString() { return toStringHelper(this) .add("id", id) .add("maxBytes", maxBytes) .add("freeBytes", getFreeBytes()) .add("reservedBytes", reservedBytes) .add("reservedRevocableBytes", reservedRevocableBytes) .add("future", future) .toString(); }
public synchronized void freeRevocable(QueryId queryId, long bytes) { checkArgument(bytes >= 0, "bytes is negative"); checkArgument(reservedRevocableBytes >= bytes, "tried to free more revocable memory than is reserved"); if (bytes == 0) { // Freeing zero bytes is a no-op return; } Long queryReservation = queryMemoryRevocableReservations.get(queryId); requireNonNull(queryReservation, "queryReservation is null"); checkArgument(queryReservation - bytes >= 0, "tried to free more revocable memory than is reserved by query"); queryReservation -= bytes; if (queryReservation == 0) { queryMemoryRevocableReservations.remove(queryId); } else { queryMemoryRevocableReservations.put(queryId, queryReservation); } reservedRevocableBytes -= bytes; if (getFreeBytes() > 0 && future != null) { future.set(null); future = null; } }
public synchronized void free(QueryId queryId, String allocationTag, long bytes) { checkArgument(bytes >= 0, "bytes is negative"); checkArgument(reservedBytes >= bytes, "tried to free more memory than is reserved"); if (bytes == 0) { // Freeing zero bytes is a no-op return; } Long queryReservation = queryMemoryReservations.get(queryId); requireNonNull(queryReservation, "queryReservation is null"); checkArgument(queryReservation - bytes >= 0, "tried to free more memory than is reserved by query"); queryReservation -= bytes; if (queryReservation == 0) { queryMemoryReservations.remove(queryId); taggedMemoryAllocations.remove(queryId); } else { queryMemoryReservations.put(queryId, queryReservation); updateTaggedMemoryAllocations(queryId, allocationTag, -bytes); } reservedBytes -= bytes; if (getFreeBytes() > 0 && future != null) { future.set(null); future = null; } }
private boolean memoryRevokingNeeded(MemoryPool memoryPool) { return memoryPool.getReservedRevocableBytes() > 0 && memoryPool.getFreeBytes() <= memoryPool.getMaxBytes() * (1.0 - memoryRevokingThreshold); }
/** * Try to reserve the given number of bytes. Return value indicates whether the caller may use the requested memory. */ public boolean tryReserve(QueryId queryId, String allocationTag, long bytes) { checkArgument(bytes >= 0, "bytes is negative"); synchronized (this) { if (getFreeBytes() - bytes < 0) { return false; } reservedBytes += bytes; if (bytes != 0) { queryMemoryReservations.merge(queryId, bytes, Long::sum); updateTaggedMemoryAllocations(queryId, allocationTag, bytes); } } onMemoryReserved(); return true; }
public ListenableFuture<?> reserveRevocable(QueryId queryId, long bytes) { checkArgument(bytes >= 0, "bytes is negative"); ListenableFuture<?> result; synchronized (this) { if (bytes != 0) { queryMemoryRevocableReservations.merge(queryId, bytes, Long::sum); } reservedRevocableBytes += bytes; if (getFreeBytes() <= 0) { if (future == null) { future = NonCancellableMemoryFuture.create(); } checkState(!future.isDone(), "future is already completed"); result = future; } else { result = NOT_BLOCKED; } } onMemoryReserved(); return result; }
private void requestMemoryRevoking(MemoryPool memoryPool, Collection<SqlTask> sqlTasks) { long remainingBytesToRevoke = (long) (-memoryPool.getFreeBytes() + (memoryPool.getMaxBytes() * (1.0 - memoryRevokingTarget))); remainingBytesToRevoke -= getMemoryAlreadyBeingRevoked(sqlTasks, memoryPool); requestRevoking(memoryPool, sqlTasks, remainingBytesToRevoke); }
/** * Reserves the given number of bytes. Caller should wait on the returned future, before allocating more memory. */ public ListenableFuture<?> reserve(QueryId queryId, String allocationTag, long bytes) { checkArgument(bytes >= 0, "bytes is negative"); ListenableFuture<?> result; synchronized (this) { if (bytes != 0) { queryMemoryReservations.merge(queryId, bytes, Long::sum); updateTaggedMemoryAllocations(queryId, allocationTag, bytes); } reservedBytes += bytes; if (getFreeBytes() <= 0) { if (future == null) { future = NonCancellableMemoryFuture.create(); } checkState(!future.isDone(), "future is already completed"); result = future; } else { result = NOT_BLOCKED; } } onMemoryReserved(); return result; }
@Test public void testBlockingOnRevocableMemoryFreeUser() { setupConsumeRevocableMemory(ONE_BYTE, 10); assertTrue(userPool.tryReserve(fakeQueryId, "test", TEN_MEGABYTES_WITHOUT_TWO_BYTES.toBytes())); // we expect 2 iterations as we have 2 bytes remaining in memory pool and we allocate 1 byte per page assertEquals(runDriversUntilBlocked(waitingForRevocableSystemMemory()), 2); assertTrue(userPool.getFreeBytes() <= 0, String.format("Expected empty pool but got [%d]", userPool.getFreeBytes())); // lets free 5 bytes userPool.free(fakeQueryId, "test", 5); assertEquals(runDriversUntilBlocked(waitingForRevocableSystemMemory()), 5); assertTrue(userPool.getFreeBytes() <= 0, String.format("Expected empty pool but got [%d]", userPool.getFreeBytes())); // 3 more bytes is enough for driver to finish userPool.free(fakeQueryId, "test", 3); assertDriversProgress(waitingForRevocableSystemMemory()); assertEquals(userPool.getFreeBytes(), 10); }
@Test public void testBlockingOnUserMemory() { setUpCountStarFromOrdersWithJoin(); assertTrue(userPool.tryReserve(fakeQueryId, "test", TEN_MEGABYTES.toBytes())); runDriversUntilBlocked(waitingForUserMemory()); assertTrue(userPool.getFreeBytes() <= 0, String.format("Expected empty pool but got [%d]", userPool.getFreeBytes())); userPool.free(fakeQueryId, "test", TEN_MEGABYTES.toBytes()); assertDriversProgress(waitingForUserMemory()); }
@Test public void testTrySetZeroBytesFullPool() { LocalMemoryContext localMemoryContext = operatorContext.localUserMemoryContext(); // fill up the pool memoryPool.reserve(new QueryId("test_query"), "test", memoryPool.getFreeBytes()); // try to reserve 0 bytes in the full pool assertTrue(localMemoryContext.trySetBytes(localMemoryContext.getBytes())); }
@Test public void testMoveQuery() { QueryId testQuery = new QueryId("test_query"); MemoryPool pool1 = new MemoryPool(new MemoryPoolId("test"), new DataSize(1000, BYTE)); MemoryPool pool2 = new MemoryPool(new MemoryPoolId("test"), new DataSize(1000, BYTE)); pool1.reserve(testQuery, "test_tag", 10); Map<String, Long> allocations = pool1.getTaggedMemoryAllocations().get(testQuery); assertEquals(allocations, ImmutableMap.of("test_tag", 10L)); pool1.moveQuery(testQuery, pool2); assertNull(pool1.getTaggedMemoryAllocations().get(testQuery)); allocations = pool2.getTaggedMemoryAllocations().get(testQuery); assertEquals(allocations, ImmutableMap.of("test_tag", 10L)); assertEquals(pool1.getFreeBytes(), 1000); assertEquals(pool2.getFreeBytes(), 990); pool2.free(testQuery, "test", 10); assertEquals(pool2.getFreeBytes(), 1000); }
private void testNoLeak(@Language("SQL") String query) throws Exception { Map<String, String> properties = ImmutableMap.<String, String>builder() .put("task.verbose-stats", "true") .build(); try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) { executor.submit(() -> queryRunner.execute(query)).get(); for (BasicQueryInfo info : queryRunner.getCoordinator().getQueryManager().getQueries()) { assertEquals(info.getState(), FINISHED); } // Make sure we didn't leak any memory on the workers for (TestingPrestoServer worker : queryRunner.getServers()) { Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent()); assertEquals(reserved.get().getMaxBytes(), reserved.get().getFreeBytes()); MemoryPool general = worker.getLocalMemoryManager().getGeneralPool(); assertEquals(general.getMaxBytes(), general.getFreeBytes()); } } }
assertTrue(reserved.get().getFreeBytes() > 0);
assertTrue(general.getFreeBytes() > 0);
@Test public void testBlockingOnRevocableMemoryFreeViaRevoke() { RevocableMemoryOperator revocableMemoryOperator = setupConsumeRevocableMemory(ONE_BYTE, 5); assertTrue(userPool.tryReserve(fakeQueryId, "test", TEN_MEGABYTES_WITHOUT_TWO_BYTES.toBytes())); // we expect 2 iterations as we have 2 bytes remaining in memory pool and we allocate 1 byte per page assertEquals(runDriversUntilBlocked(waitingForRevocableSystemMemory()), 2); revocableMemoryOperator.getOperatorContext().requestMemoryRevoking(); // 2 more iterations assertEquals(runDriversUntilBlocked(waitingForRevocableSystemMemory()), 2); revocableMemoryOperator.getOperatorContext().requestMemoryRevoking(); // 3 more bytes is enough for driver to finish assertDriversProgress(waitingForRevocableSystemMemory()); assertEquals(userPool.getFreeBytes(), 2); }
assertTrue(reserved.get().getFreeBytes() > 0); Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent()); assertEquals(reserved.get().getMaxBytes(), reserved.get().getFreeBytes()); MemoryPool general = worker.getLocalMemoryManager().getGeneralPool(); assertEquals(general.getMaxBytes(), general.getFreeBytes());
assertEquals(10, memoryPool.getFreeBytes()); assertMemoryRevokingNotRequested(); assertEquals(1, memoryPool.getFreeBytes()); requestMemoryRevoking(scheduler); assertEquals(-6, memoryPool.getFreeBytes()); requestMemoryRevoking(scheduler); requestMemoryRevoking(scheduler); assertMemoryRevokingRequestedFor(operatorContext3); assertEquals(-3, memoryPool.getFreeBytes()); assertEquals(-6, memoryPool.getFreeBytes()); requestMemoryRevoking(scheduler); assertEquals(-7, memoryPool.getFreeBytes()); requestMemoryRevoking(scheduler);
long reservedMemoryInBytes = memoryPool.getFreeBytes() - additionalMemoryInBytes; memoryPool.reserve(queryId, "test", reservedMemoryInBytes);
private void testNoLeak(@Language("SQL") String query) throws Exception { Map<String, String> properties = ImmutableMap.<String, String>builder() .put("task.verbose-stats", "true") .build(); try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) { executor.submit(() -> queryRunner.execute(query)).get(); for (BasicQueryInfo info : queryRunner.getCoordinator().getQueryManager().getQueries()) { assertEquals(info.getState(), FINISHED); } // Make sure we didn't leak any memory on the workers for (TestingPrestoServer worker : queryRunner.getServers()) { Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool(); assertTrue(reserved.isPresent()); assertEquals(reserved.get().getMaxBytes(), reserved.get().getFreeBytes()); MemoryPool general = worker.getLocalMemoryManager().getGeneralPool(); assertEquals(general.getMaxBytes(), general.getFreeBytes()); } } }