public StageId(String queryId, int id) { this(new QueryId(queryId), id); }
public static StageId valueOf(List<String> ids) { checkArgument(ids.size() == 2, "Expected two ids but got: %s", ids); return new StageId(new QueryId(ids.get(0)), Integer.parseInt(ids.get(1))); }
@JsonCreator public static QueryId valueOf(String queryId) { List<String> ids = parseDottedId(queryId, 1, "queryId"); return new QueryId(ids.get(0)); }
public QueryId getQueryId() { return new QueryId(QueryId.parseDottedId(fullId, 3, "taskId").get(0)); }
@GET @Path("{queryId}") @Produces(MediaType.APPLICATION_JSON) public QueryStateInfo getQueryStateInfo(@PathParam("queryId") String queryId) throws WebApplicationException { try { return getQueryStateInfo(queryManager.getQueryInfo(new QueryId(queryId))); } catch (NoSuchElementException e) { throw new WebApplicationException(NOT_FOUND); } } }
static List<LowMemoryKiller.QueryMemoryInfo> toQueryMemoryInfoList(String reservedQuery, Map<String, Map<String, Long>> queries) { ImmutableList.Builder<LowMemoryKiller.QueryMemoryInfo> result = ImmutableList.builder(); for (Map.Entry<String, Map<String, Long>> entry : queries.entrySet()) { String queryId = entry.getKey(); long totalReservation = entry.getValue().values().stream() .mapToLong(x -> x) .sum(); result.add(new LowMemoryKiller.QueryMemoryInfo(new QueryId(queryId), queryId.equals(reservedQuery) ? RESERVED_POOL : GENERAL_POOL, totalReservation)); } return result.build(); }
@Test public void testSkewedQuery() { int reservePool = 10; int generalPool = 12; // q2 is the query with the most total memory reservation, but not the query with the max memory reservation. // This also tests the corner case where a node doesn't have a general pool. Map<String, Map<String, Long>> queries = ImmutableMap.<String, Map<String, Long>>builder() .put("q_1", ImmutableMap.of("n1", 0L, "n2", 8L, "n3", 0L, "n4", 0L, "n5", 0L)) .put("q_2", ImmutableMap.of("n1", 3L, "n2", 5L, "n3", 2L, "n4", 4L, "n5", 0L)) .put("q_3", ImmutableMap.of("n1", 0L, "n2", 0L, "n3", 9L, "n4", 0L, "n5", 0L)) .put("q_r", ImmutableMap.of("n1", 6L, "n2", 6L, "n3", 6L, "n4", 6L, "n5", 6L)) .build(); assertEquals( lowMemoryKiller.chooseQueryToKill( toQueryMemoryInfoList("q_r", queries), toNodeMemoryInfoList(reservePool, generalPool, "q_r", queries)), Optional.of(new QueryId("q_2"))); } }
@Test public void testSkewedQuery() { int reservePool = 10; int generalPool = 12; // q1 is neither the query with the most total memory reservation, nor the query with the max memory reservation. // This also tests the corner case where a node doesn't have a general pool. Map<String, Map<String, Long>> queries = ImmutableMap.<String, Map<String, Long>>builder() .put("q_1", ImmutableMap.of("n1", 0L, "n2", 8L, "n3", 0L, "n4", 0L, "n5", 0L)) .put("q_2", ImmutableMap.of("n1", 3L, "n2", 5L, "n3", 2L, "n4", 4L, "n5", 0L)) .put("q_3", ImmutableMap.of("n1", 0L, "n2", 0L, "n3", 9L, "n4", 0L, "n5", 0L)) .put("q_r", ImmutableMap.of("n1", 6L, "n2", 6L, "n3", 6L, "n4", 6L, "n5", 6L)) .build(); assertEquals( lowMemoryKiller.chooseQueryToKill( toQueryMemoryInfoList("q_r", queries), toNodeMemoryInfoList(reservePool, generalPool, "q_r", queries)), Optional.of(new QueryId("q_1"))); } }
public MockQueryExecution(long memoryUsage, String queryId, int priority, Duration cpuUsage) { this.memoryUsage = succinctBytes(memoryUsage); this.cpuUsage = cpuUsage; this.session = testSessionBuilder() .setSystemProperty(QUERY_PRIORITY, String.valueOf(priority)) .build(); this.resourceGroupId = Optional.empty(); this.queryId = new QueryId(queryId); }
@Test public void testTrySetZeroBytesFullPool() { LocalMemoryContext localMemoryContext = operatorContext.localUserMemoryContext(); // fill up the pool memoryPool.reserve(new QueryId("test_query"), "test", memoryPool.getFreeBytes()); // try to reserve 0 bytes in the full pool assertTrue(localMemoryContext.trySetBytes(localMemoryContext.getBytes())); }
@Test public void testTaggedAllocations() { QueryId testQuery = new QueryId("test_query"); MemoryPool testPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(1000, BYTE)); testPool.reserve(testQuery, "test_tag", 10); Map<String, Long> allocations = testPool.getTaggedMemoryAllocations().get(testQuery); assertEquals(allocations, ImmutableMap.of("test_tag", 10L)); // free 5 bytes for test_tag testPool.free(testQuery, "test_tag", 5); assertEquals(allocations, ImmutableMap.of("test_tag", 5L)); testPool.reserve(testQuery, "test_tag2", 20); assertEquals(allocations, ImmutableMap.of("test_tag", 5L, "test_tag2", 20L)); // free the remaining 5 bytes for test_tag testPool.free(testQuery, "test_tag", 5); assertEquals(allocations, ImmutableMap.of("test_tag2", 20L)); // free all for test_tag2 testPool.free(testQuery, "test_tag2", 20); assertEquals(testPool.getTaggedMemoryAllocations().size(), 0); }
@Test public void testLeakDetector() { QueryId testQuery = new QueryId("test"); ClusterMemoryLeakDetector leakDetector = new ClusterMemoryLeakDetector(); leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(), ImmutableMap.of()); assertEquals(leakDetector.getNumberOfLeakedQueries(), 0); // the leak detector should report no leaked queries as the query is still running leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.getId(), RUNNING)), ImmutableMap.of(testQuery, 1L)); assertEquals(leakDetector.getNumberOfLeakedQueries(), 0); // the leak detector should report exactly one leaked query since the query is finished, and its end time is way in the past leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.getId(), FINISHED)), ImmutableMap.of(testQuery, 1L)); assertEquals(leakDetector.getNumberOfLeakedQueries(), 1); // the leak detector should report no leaked queries as the query doesn't have any memory reservation leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.getId(), FINISHED)), ImmutableMap.of(testQuery, 0L)); assertEquals(leakDetector.getNumberOfLeakedQueries(), 0); // the leak detector should report exactly one leaked query since the coordinator doesn't know of any query leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(), ImmutableMap.of(testQuery, 1L)); assertEquals(leakDetector.getNumberOfLeakedQueries(), 1); }
@Test public void testMoveQuery() { QueryId testQuery = new QueryId("test_query"); MemoryPool pool1 = new MemoryPool(new MemoryPoolId("test"), new DataSize(1000, BYTE)); MemoryPool pool2 = new MemoryPool(new MemoryPoolId("test"), new DataSize(1000, BYTE)); pool1.reserve(testQuery, "test_tag", 10); Map<String, Long> allocations = pool1.getTaggedMemoryAllocations().get(testQuery); assertEquals(allocations, ImmutableMap.of("test_tag", 10L)); pool1.moveQuery(testQuery, pool2); assertNull(pool1.getTaggedMemoryAllocations().get(testQuery)); allocations = pool2.getTaggedMemoryAllocations().get(testQuery); assertEquals(allocations, ImmutableMap.of("test_tag", 10L)); assertEquals(pool1.getFreeBytes(), 1000); assertEquals(pool2.getFreeBytes(), 990); pool2.free(testQuery, "test", 10); assertEquals(pool2.getFreeBytes(), 1000); }
private SqlTask newSqlTask() { TaskId taskId = new TaskId("query", 0, idGeneator.incrementAndGet()); URI location = URI.create("fake://task/" + taskId); return createSqlTask( taskId, location, "fake", new DefaultQueryContext(new QueryId("query"), new DataSize(1, MEGABYTE), new DataSize(2, MEGABYTE), memoryPool, new TestingGcMonitor(), executor, scheduledExecutor, new DataSize(1, GIGABYTE), spillSpaceTracker), sqlTaskExecutionFactory, executor, Functions.identity(), new DataSize(32, MEGABYTE), new CounterStat()); } }
private SqlStageExecution createSqlStageExecution(StageExecutionPlan tableScanPlan, NodeTaskMap nodeTaskMap) { StageId stageId = new StageId(new QueryId("query"), 0); SqlStageExecution stage = SqlStageExecution.createSqlStageExecution(stageId, locationFactory.createStageLocation(stageId), tableScanPlan.getFragment(), new MockRemoteTaskFactory(queryExecutor, scheduledExecutor), TEST_SESSION, true, nodeTaskMap, queryExecutor, new NoOpFailureDetector(), new SplitSchedulerStats()); stage.setOutputBuffers(createInitialEmptyOutputBuffers(PARTITIONED) .withBuffer(OUT, 0) .withNoMoreBufferIds()); return stage; }
public static Session toSession(ConnectorTransactionHandle transactionHandle, ConnectorSession session) { TransactionId transactionId = ((GlobalSystemTransactionHandle) transactionHandle).getTransactionId(); return Session.builder(new SessionPropertyManager(SYSTEM_SESSION_PROPERTIES)) .setQueryId(new QueryId(session.getQueryId())) .setTransactionId(transactionId) .setCatalog("catalog") .setSchema("schema") .setPath(new SqlPath(Optional.of("path"))) .setIdentity(session.getIdentity()) .setTimeZoneKey(session.getTimeZoneKey()) .setLocale(session.getLocale()) .setStartTime(session.getStartTime()) .build(); } }
private void setUp(Supplier<List<Driver>> driversSupplier) { checkState(localQueryRunner == null, "Already set up"); Session session = testSessionBuilder() .setCatalog("tpch") .setSchema("tiny") .setSystemProperty("task_default_concurrency", "1") .build(); localQueryRunner = queryRunnerWithInitialTransaction(session); // add tpch localQueryRunner.createCatalog("tpch", new TpchConnectorFactory(1), ImmutableMap.of()); userPool = new MemoryPool(new MemoryPoolId("test"), TEN_MEGABYTES); fakeQueryId = new QueryId("fake"); SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(new DataSize(1, GIGABYTE)); DefaultQueryContext queryContext = new DefaultQueryContext(new QueryId("query"), TEN_MEGABYTES, new DataSize(20, MEGABYTE), userPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), TEN_MEGABYTES, spillSpaceTracker); taskContext = createTaskContext(queryContext, localQueryRunner.getExecutor(), session); drivers = driversSupplier.get(); }
@BeforeMethod public void setUpTest() { memoryPool = new MemoryPool(new MemoryPoolId("test"), memoryPoolSize); queryContext = new DefaultQueryContext( new QueryId("test_query"), queryMaxMemory, queryMaxTotalMemory, memoryPool, new TestingGcMonitor(), notificationExecutor, yieldExecutor, queryMaxSpillSize, spillSpaceTracker); taskContext = queryContext.addTaskContext( new TaskStateMachine(new TaskId("query", 0, 0), notificationExecutor), testSessionBuilder().build(), true, true, OptionalInt.empty()); pipelineContext = taskContext.addPipelineContext(0, true, true, false); driverContext = pipelineContext.addDriverContext(); operatorContext = driverContext.addOperatorContext(1, new PlanNodeId("a"), "test-operator"); }
private TaskContext newTestingTaskContext(ScheduledExecutorService taskNotificationExecutor, ScheduledExecutorService driverYieldExecutor, TaskStateMachine taskStateMachine) { DefaultQueryContext queryContext = new DefaultQueryContext( new QueryId("queryid"), new DataSize(1, MEGABYTE), new DataSize(2, MEGABYTE), new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE)), new TestingGcMonitor(), taskNotificationExecutor, driverYieldExecutor, new DataSize(1, MEGABYTE), new SpillSpaceTracker(new DataSize(1, GIGABYTE))); return queryContext.addTaskContext(taskStateMachine, TEST_SESSION, false, false, OptionalInt.empty()); }
@Test(expectedExceptions = PrestoException.class) public void testInvalidTimeZone() { HttpServletRequest request = new MockHttpServletRequest( ImmutableListMultimap.<String, String>builder() .put(PRESTO_USER, "testUser") .put(PRESTO_TIME_ZONE, "unknown_timezone") .build(), "testRemote"); HttpRequestSessionContext context = new HttpRequestSessionContext(request); QuerySessionSupplier sessionSupplier = new QuerySessionSupplier( createTestTransactionManager(), new AllowAllAccessControl(), new SessionPropertyManager(), new SqlEnvironmentConfig()); sessionSupplier.createSession(new QueryId("test_query_id"), context); }