private static URI findCancelableLeafStage(QueryInfo queryInfo) { // if query is running, find the leaf-most running stage return queryInfo.getOutputStage().map(Query::findCancelableLeafStage).orElse(null); }
private static List<StageCpuDistribution> getCpuDistributions(QueryInfo queryInfo) { if (!queryInfo.getOutputStage().isPresent()) { return ImmutableList.of(); } ImmutableList.Builder<StageCpuDistribution> builder = ImmutableList.builder(); populateDistribution(queryInfo.getOutputStage().get(), builder); return builder.build(); }
private Optional<String> createTextQueryPlan(QueryInfo queryInfo) { try { if (queryInfo.getOutputStage().isPresent()) { return Optional.of(textDistributedPlan( queryInfo.getOutputStage().get(), functionRegistry, queryInfo.getSession().toSession(sessionPropertyManager), false)); } } catch (Exception e) { // Sometimes it is expected to fail. For example if generated plan is too long. // Don't fail to create event if the plan can not be created. log.warn(e, "Error creating explain plan for query %s", queryInfo.getQueryId()); } return Optional.empty(); }
@Override public Page getOutput() { if (!finishing) { return null; } QueryInfo queryInfo = queryPerformanceFetcher.getQueryInfo(operatorContext.getDriverContext().getTaskId().getQueryId()); checkState(queryInfo.getOutputStage().isPresent(), "Output stage is missing"); checkState(queryInfo.getOutputStage().get().getSubStages().size() == 1, "Expected one sub stage of explain node"); if (!hasFinalStageInfo(queryInfo.getOutputStage().get())) { return null; } String plan = textDistributedPlan(queryInfo.getOutputStage().get().getSubStages().get(0), functionRegistry, operatorContext.getSession(), verbose); BlockBuilder builder = VARCHAR.createBlockBuilder(null, 1); VARCHAR.writeString(builder, plan); outputConsumed = true; return new Page(builder.build()); }
private synchronized void closeExchangeClientIfNecessary(QueryInfo queryInfo) { // Close the exchange client if the query has failed, or if the query // is done and it does not have an output stage. The latter happens // for data definition executions, as those do not have output. if ((queryInfo.getState() == FAILED) || (queryInfo.getState().isDone() && !queryInfo.getOutputStage().isPresent())) { exchangeClient.close(); } }
List<StageInfo> stages = StageInfo.getAllStages(queryInfo.getOutputStage());
public void queryCompletedEvent(QueryInfo queryInfo) { QueryStats queryStats = queryInfo.getQueryStats(); eventListenerManager.queryCompleted( new QueryCompletedEvent( createQueryMetadata(queryInfo), createQueryStatistics(queryInfo), createQueryContext(queryInfo.getSession(), queryInfo.getResourceGroupId()), getQueryIOMetadata(queryInfo), createQueryFailureInfo(queryInfo.getFailureInfo(), queryInfo.getOutputStage()), queryInfo.getWarnings(), ofEpochMilli(queryStats.getCreateTime().getMillis()), ofEpochMilli(queryStats.getExecutionStartTime().getMillis()), ofEpochMilli(queryStats.getEndTime() != null ? queryStats.getEndTime().getMillis() : 0))); logQueryTimeline(queryInfo); }
private QueryMetadata createQueryMetadata(QueryInfo queryInfo) { return new QueryMetadata( queryInfo.getQueryId().toString(), queryInfo.getSession().getTransactionId().map(TransactionId::toString), queryInfo.getQuery(), queryInfo.getState().toString(), queryInfo.getSelf(), createTextQueryPlan(queryInfo), queryInfo.getOutputStage().flatMap(stage -> stageInfoCodec.toJsonWithLengthLimit(stage, maxJsonLimit))); }
public void pruneQueryInfo() if (!finalInfo.isPresent() || !finalInfo.get().getOutputStage().isPresent()) { return; Optional<StageInfo> prunedOutputStage = queryInfo.getOutputStage().map(outputStage -> new StageInfo( outputStage.getStageId(), outputStage.getState(),
if ((queryInfo.getState() == QueryState.FINISHED) && !queryInfo.getOutputStage().isPresent()) { columns = ImmutableList.of(new Column("result", BooleanType.BOOLEAN)); data = ImmutableSet.of(ImmutableList.of(true));
assertEquals(queryInfo.getQueryId(), TEST_SESSION.getQueryId()); assertEquals(queryInfo.getSelf(), LOCATION); assertFalse(queryInfo.getOutputStage().isPresent()); assertEquals(queryInfo.getQuery(), QUERY); assertEquals(queryInfo.getInputs(), INPUTS);
private static StatementStats toStatementStats(QueryInfo queryInfo) { QueryStats queryStats = queryInfo.getQueryStats(); StageInfo outputStage = queryInfo.getOutputStage().orElse(null); return StatementStats.builder() .setState(queryInfo.getState().toString()) .setQueued(queryInfo.getState() == QueryState.QUEUED) .setScheduled(queryInfo.isScheduled()) .setNodes(globalUniqueNodes(outputStage).size()) .setTotalSplits(queryStats.getTotalDrivers()) .setQueuedSplits(queryStats.getQueuedDrivers()) .setRunningSplits(queryStats.getRunningDrivers() + queryStats.getBlockedDrivers()) .setCompletedSplits(queryStats.getCompletedDrivers()) .setCpuTimeMillis(queryStats.getTotalCpuTime().toMillis()) .setWallTimeMillis(queryStats.getTotalScheduledTime().toMillis()) .setQueuedTimeMillis(queryStats.getQueuedTime().toMillis()) .setElapsedTimeMillis(queryStats.getElapsedTime().toMillis()) .setProcessedRows(queryStats.getRawInputPositions()) .setProcessedBytes(queryStats.getRawInputDataSize().toBytes()) .setPeakMemoryBytes(queryStats.getPeakUserMemoryReservation().toBytes()) .setSpilledBytes(queryStats.getSpilledDataSize().toBytes()) .setRootStage(toStageStats(outputStage)) .build(); }
private static URI findCancelableLeafStage(QueryInfo queryInfo) { if (queryInfo.getOutputStage() == null) { // query is not running yet, cannot cancel leaf stage return null; } // query is running, find the leaf-most running stage return findCancelableLeafStage(queryInfo.getOutputStage()); }
private static List<Column> createColumnsList(QueryInfo queryInfo) { requireNonNull(queryInfo, "queryInfo is null"); StageInfo outputStage = queryInfo.getOutputStage(); requireNonNull(outputStage, "outputStage is null"); List<String> names = queryInfo.getFieldNames(); List<Type> types = outputStage.getTypes(); checkArgument(names.size() == types.size(), "names and types size mismatch"); ImmutableList.Builder<Column> list = ImmutableList.builder(); for (int i = 0; i < names.size(); i++) { String name = names.get(i); TypeSignature typeSignature = types.get(i).getTypeSignature(); String type = typeSignature.toString(); list.add(new Column(name, type, new ClientTypeSignature(typeSignature))); } return list.build(); }
private static boolean allQueriesBlocked(List<QueryInfo> current) { boolean allDriversBlocked = current.stream() .flatMap(query -> getAllStages(query.getOutputStage()).stream()) .flatMap(stage -> stage.getTasks().stream()) .flatMap(task -> task.getStats().getPipelines().stream()) .flatMap(pipeline -> pipeline.getDrivers().stream()) .allMatch(DriverStats::isFullyBlocked); boolean waitingForMemory = current.stream().allMatch(TestMemoryManager::atLeastOneOperatorWaitingForMemory); return allDriversBlocked && waitingForMemory; }
private static boolean atLeastOneOperatorWaitingForMemory(QueryInfo query) { return getAllStages(query.getOutputStage()).stream() .flatMap(stage -> stage.getTasks().stream()) .map(TaskInfo::getStats) .anyMatch(task -> task.getBlockedReasons().contains(WAITING_FOR_MEMORY)); }
if (queryInfo == null || queryInfo.getOutputStage() == null) { return; queryInfo.getOutputStage().getStageId(), queryInfo.getOutputStage().getState(), queryInfo.getOutputStage().getSelf(), queryInfo.getOutputStage().getTypes(), queryInfo.getOutputStage().getStageStats(), queryInfo.getOutputStage().getFailureCause() );
if (!isQueryStarted(queryInfo) || queryInfo.getOutputStage() == null) { return null; List<Type> types = queryInfo.getOutputStage().getTypes(); updateExchangeClient(queryInfo.getOutputStage());
Optional<TaskInfo> task = findFailedTask(queryInfo.getOutputStage()); String failureHost = task.map(x -> x.getSelf().getHost()).orElse(null); String failureTask = task.map(x -> x.getTaskId().toString()).orElse(null); failureTask, failureHost, objectMapper.writeValueAsString(queryInfo.getOutputStage()), objectMapper.writeValueAsString(queryInfo.getFailureInfo()), objectMapper.writeValueAsString(queryInfo.getInputs()),
private static StatementStats toStatementStats(QueryInfo queryInfo) { QueryStats queryStats = queryInfo.getQueryStats(); return StatementStats.builder() .setState(queryInfo.getState().toString()) .setScheduled(queryInfo.isScheduled()) .setNodes(globalUniqueNodes(queryInfo.getOutputStage()).size()) .setTotalSplits(queryStats.getTotalDrivers()) .setQueuedSplits(queryStats.getQueuedDrivers()) .setRunningSplits(queryStats.getRunningDrivers()) .setCompletedSplits(queryStats.getCompletedDrivers()) .setUserTimeMillis(queryStats.getTotalUserTime().toMillis()) .setCpuTimeMillis(queryStats.getTotalCpuTime().toMillis()) .setWallTimeMillis(queryStats.getTotalScheduledTime().toMillis()) .setProcessedRows(queryStats.getRawInputPositions()) .setProcessedBytes(queryStats.getRawInputDataSize().toBytes()) .setRootStage(toStageStats(queryInfo.getOutputStage())) .build(); }