public TransactionInfo getTransactionInfo() { Duration idleTime = Optional.ofNullable(idleStartTime.get()) .map(Duration::nanosSince) .orElse(new Duration(0, MILLISECONDS)); // dereferencing this field is safe because the field is atomic @SuppressWarnings("FieldAccessNotGuarded") Optional<ConnectorId> writtenConnectorId = Optional.ofNullable(this.writtenConnectorId.get()); // copying the key set is safe here because the map is concurrent @SuppressWarnings("FieldAccessNotGuarded") List<ConnectorId> connectorIds = ImmutableList.copyOf(connectorIdToMetadata.keySet()); return new TransactionInfo(transactionId, isolationLevel, readOnly, autoCommitContext, createTime, idleTime, connectorIds, writtenConnectorId); }
private static Duration randomizeWaitTime(Duration waitTime) { // Randomize in [T/2, T], so wait is not near zero and the client-supplied max wait time is respected long halfWaitMillis = waitTime.toMillis() / 2; return new Duration(halfWaitMillis + ThreadLocalRandom.current().nextLong(halfWaitMillis), MILLISECONDS); } }
@GET @Produces(APPLICATION_JSON) public ServerInfo getInfo() { boolean starting = !catalogStore.areCatalogsLoaded(); return new ServerInfo(version, environment, coordinator, starting, Optional.of(nanosSince(startTime))); }
@Override public TableWriterInfo mergeWith(TableWriterInfo other) { return new TableWriterInfo( Math.max(pageSinkPeakMemoryUsage, other.pageSinkPeakMemoryUsage), succinctNanos(statisticsWallTime.roundTo(NANOSECONDS) + other.statisticsWallTime.roundTo(NANOSECONDS)), succinctNanos(statisticsCpuTime.roundTo(NANOSECONDS) + other.statisticsCpuTime.roundTo(NANOSECONDS)), succinctNanos(validationCpuTime.roundTo(NANOSECONDS) + other.validationCpuTime.roundTo(NANOSECONDS))); }
public boolean isExpired(Duration idleTimeout) { Long idleStartTime = this.idleStartTime.get(); return idleStartTime != null && Duration.nanosSince(idleStartTime).compareTo(idleTimeout) > 0; }
protected void configureGroup(ResourceGroup group, ResourceGroupSpec match) if (match.getSoftMemoryLimit().isPresent()) { group.setSoftMemoryLimit(match.getSoftMemoryLimit().get()); double fraction = match.getSoftMemoryLimitFraction().get(); generalPoolMemoryFraction.put(group, fraction); group.setSoftMemoryLimit(new DataSize(generalPoolBytes * fraction, BYTE)); limit = match.getSoftCpuLimit().get(); long rate = (long) Math.min(1000.0 * limit.toMillis() / (double) getCpuQuotaPeriod().get().toMillis(), Long.MAX_VALUE); rate = Math.max(1, rate); group.setCpuQuotaGenerationMillisPerSecond(rate);
private void splitCompletedEvent(TaskId taskId, DriverStats driverStats, @Nullable String failureType, @Nullable String failureMessage) Optional<Duration> timeToStart = Optional.empty(); if (driverStats.getStartTime() != null) { timeToStart = Optional.of(ofMillis(driverStats.getStartTime().getMillis() - driverStats.getCreateTime().getMillis())); Optional<Duration> timeToEnd = Optional.empty(); if (driverStats.getEndTime() != null) { timeToEnd = Optional.of(ofMillis(driverStats.getEndTime().getMillis() - driverStats.getCreateTime().getMillis())); Optional.ofNullable(driverStats.getEndTime()).map(endTime -> endTime.toDate().toInstant()), new SplitStatistics( ofMillis(driverStats.getTotalCpuTime().toMillis()), ofMillis(driverStats.getElapsedTime().toMillis()), ofMillis(driverStats.getQueuedTime().toMillis()), ofMillis(driverStats.getRawInputReadTime().toMillis()), driverStats.getRawInputPositions(), driverStats.getRawInputDataSize().toBytes(), timeToStart, timeToEnd),
queryRunner, newSessionWithResourceEstimates(new ResourceEstimates( Optional.of(Duration.valueOf("4m")), Optional.empty(), Optional.of(DataSize.valueOf("400MB")))), LONG_LASTING_QUERY, createResourceGroupId("global", "small")); queryRunner, newSessionWithResourceEstimates(new ResourceEstimates( Optional.of(Duration.valueOf("4m")), Optional.empty(), Optional.of(DataSize.valueOf("600MB")))), LONG_LASTING_QUERY, createResourceGroupId("global", "other")); queryRunner, newSessionWithResourceEstimates(new ResourceEstimates( Optional.of(Duration.valueOf("4m")), queryRunner, newSessionWithResourceEstimates(new ResourceEstimates( Optional.of(Duration.valueOf("1s")), Optional.of(Duration.valueOf("1s")), Optional.of(DataSize.valueOf("6TB")))), LONG_LASTING_QUERY, createResourceGroupId("global", "huge_memory")); queryRunner,
@Test public void testCreateTableAsSelectDifferentCatalog() throws Exception { handle.execute("CREATE TABLE \"my_test_table2\" (column1 BIGINT, column2 DOUBLE)"); SqlParser parser = new SqlParser(); Query query = new Query(CATALOG, SCHEMA, ImmutableList.of(), "CREATE TABLE public.my_test_table2 AS SELECT 1 column1, 2E0 column2", ImmutableList.of(), null, null, ImmutableMap.of()); QueryRewriter rewriter = new QueryRewriter(parser, URL, QualifiedName.of("other_catalog", "other_schema", "tmp_"), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), 1, new Duration(10, SECONDS)); Query rewrittenQuery = rewriter.shadowQuery(query); assertEquals(rewrittenQuery.getPreQueries().size(), 1); CreateTableAsSelect createTableAs = (CreateTableAsSelect) parser.createStatement(rewrittenQuery.getPreQueries().get(0)); assertEquals(createTableAs.getName().getParts().size(), 3); assertEquals(createTableAs.getName().getPrefix().get(), QualifiedName.of("other_catalog", "other_schema")); assertTrue(createTableAs.getName().getSuffix().startsWith("tmp_")); assertFalse(createTableAs.getName().getSuffix().contains("my_test_table")); }
@Override public Duration getTotalCpuTime() { SqlQueryScheduler scheduler = queryScheduler.get(); Optional<QueryInfo> finalQueryInfo = stateMachine.getFinalQueryInfo(); if (finalQueryInfo.isPresent()) { return finalQueryInfo.get().getQueryStats().getTotalCpuTime(); } if (scheduler == null) { return new Duration(0, SECONDS); } return scheduler.getTotalCpuTime(); }
private void printStats(int indent, PlanNodeId planNodeId, boolean printInput, boolean printFiltered) if (!stats.isPresent()) { return; long totalScheduledMillis = stats.get().values().stream() .mapToLong(node -> node.getPlanNodeScheduledTime().toMillis()) .sum(); long totalCpuMillis = stats.get().values().stream() .mapToLong(node -> node.getPlanNodeCpuTime().toMillis()) .sum(); double scheduledTimeFraction = 100.0d * nodeStats.getPlanNodeScheduledTime().toMillis() / totalScheduledMillis; double cpuTimeFraction = 100.0d * nodeStats.getPlanNodeCpuTime().toMillis() / totalCpuMillis; nodeStats.getPlanNodeCpuTime().convertToMostSuccinctTimeUnit(), formatDouble(cpuTimeFraction), nodeStats.getPlanNodeScheduledTime().convertToMostSuccinctTimeUnit(), formatDouble(scheduledTimeFraction))); output.append(format(", Input: %s (%s)", formatPositions(nodeStats.getPlanNodeInputPositions()), nodeStats.getPlanNodeInputDataSize().toString())); nodeStats.getPlanNodeOutputDataSize().toString())); if (printFiltered) { double filtered = 100.0d * (nodeStats.getPlanNodeInputPositions() - nodeStats.getPlanNodeOutputPositions()) / nodeStats.getPlanNodeInputPositions();
@Test(timeOut = 10000) public void testAddLocation() throws Exception DataSize maxResponseSize = new DataSize(10, Unit.MEGABYTE); MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize); new DataSize(32, Unit.MEGABYTE), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), true, new TestingHttpClient(processor, newCachedThreadPool(daemonThreadsNamed("test-%s"))), exchangeClient.addLocation(location1); assertFalse(tryGetFutureValue(exchangeClient.isBlocked(), 10, MILLISECONDS).isPresent()); assertEquals(exchangeClient.isClosed(), false); assertPageEquals(getNextPage(exchangeClient), createPage(6)); assertFalse(tryGetFutureValue(exchangeClient.isBlocked(), 10, MILLISECONDS).isPresent()); assertEquals(exchangeClient.isClosed(), false);
@Test public void testConstructor() DateTime.parse("1991-09-06T05:02-05:30"), DateTime.parse("1991-09-06T06:00-05:30"), Duration.valueOf("8m"), Duration.valueOf("7m"), Duration.valueOf("34m"), Duration.valueOf("44m"), Duration.valueOf("9m"), Duration.valueOf("10m"), Duration.valueOf("11m"), Duration.valueOf("12m"), 13, 14, true, Duration.valueOf("23m"), Duration.valueOf("24m"), Duration.valueOf("26m"), true, ImmutableSet.of(BlockedReason.WAITING_FOR_MEMORY), assertEquals(basicInfo.getQueryStats().getElapsedTime(), Duration.valueOf("8m")); assertEquals(basicInfo.getQueryStats().getExecutionTime(), Duration.valueOf("44m")); assertEquals(basicInfo.getQueryStats().getTotalCpuTime(), Duration.valueOf("24m"));
@Test public void testGetServerInfo() throws Exception { ServerInfo expected = new ServerInfo(UNKNOWN, "test", true, false, Optional.of(Duration.valueOf("2m"))); server.enqueue(new MockResponse() .addHeader(CONTENT_TYPE, "application/json") .setBody(SERVER_INFO_CODEC.toJson(expected))); QueryExecutor executor = new QueryExecutor(new OkHttpClient()); ServerInfo actual = executor.getServerInfo(server.url("/v1/info").uri()); assertEquals(actual.getEnvironment(), "test"); assertEquals(actual.getUptime(), Optional.of(Duration.valueOf("2m"))); assertEquals(server.getRequestCount(), 1); assertEquals(server.takeRequest().getPath(), "/v1/info"); } }
@Test public void testAdd() assertEquals(actual.getTotalDrivers(), 3 * 1); assertEquals(actual.getAddInputCalls(), 3 * 2); assertEquals(actual.getAddInputWall(), new Duration(3 * 3, NANOSECONDS)); assertEquals(actual.getAddInputCpu(), new Duration(3 * 4, NANOSECONDS)); assertEquals(actual.getRawInputDataSize(), new DataSize(3 * 5, BYTE)); assertEquals(actual.getInputDataSize(), new DataSize(3 * 6, BYTE)); assertEquals(actual.getInputPositions(), 3 * 7); assertEquals(actual.getSumSquaredInputPositions(), 3 * 8.0); assertEquals(actual.getGetOutputCalls(), 3 * 9); assertEquals(actual.getGetOutputWall(), new Duration(3 * 10, NANOSECONDS)); assertEquals(actual.getGetOutputCpu(), new Duration(3 * 11, NANOSECONDS)); assertEquals(actual.getOutputDataSize(), new DataSize(3 * 12, BYTE)); assertEquals(actual.getOutputPositions(), 3 * 13); assertEquals(actual.getPhysicalWrittenDataSize(), new DataSize(3 * 14, BYTE)); assertEquals(actual.getBlockedWall(), new Duration(3 * 15, NANOSECONDS)); assertEquals(actual.getFinishCalls(), 3 * 16); assertEquals(actual.getFinishWall(), new Duration(3 * 17, NANOSECONDS)); assertEquals(actual.getFinishCpu(), new Duration(3 * 18, NANOSECONDS)); assertEquals(actual.getUserMemoryReservation(), new DataSize(3 * 19, BYTE)); assertEquals(actual.getRevocableMemoryReservation(), new DataSize(3 * 20, BYTE));
private void assertAllTimeSpentInQueueing(QueryState expectedState, Consumer<QueryStateMachine> stateTransition) { TestingTicker ticker = new TestingTicker(); QueryStateMachine stateMachine = createQueryStateMachineWithTicker(ticker); ticker.increment(7, MILLISECONDS); stateTransition.accept(stateMachine); assertEquals(stateMachine.getQueryState(), expectedState); QueryStats queryStats = stateMachine.getQueryInfo(Optional.empty()).getQueryStats(); assertEquals(queryStats.getQueuedTime(), new Duration(7, MILLISECONDS)); assertEquals(queryStats.getResourceWaitingTime(), new Duration(0, MILLISECONDS)); assertEquals(queryStats.getTotalPlanningTime(), new Duration(0, MILLISECONDS)); assertEquals(queryStats.getExecutionTime(), new Duration(0, MILLISECONDS)); assertEquals(queryStats.getFinishingTime(), new Duration(0, MILLISECONDS)); }