if (plan.getOrderBy().getOrderByExpressions().isEmpty()) { bestCandidates.add(plan);
@Override public Cost getCost() { Double outputBytes = this.accept(new ByteCountVisitor()); Double inputRows = this.getDelegate().accept(new RowCountVisitor()); Double rowWidth = this.accept(new AvgRowWidthVisitor()); if (inputRows == null || outputBytes == null || rowWidth == null) { return Cost.UNKNOWN; } double inputBytes = inputRows * rowWidth; double rowsBeforeHaving = RowCountVisitor.aggregate( RowCountVisitor.filter( inputRows.doubleValue(), RowCountVisitor.stripSkipScanFilter( context.getScan().getFilter())), groupBy); double rowsAfterHaving = RowCountVisitor.filter(rowsBeforeHaving, having); double bytesBeforeHaving = rowWidth * rowsBeforeHaving; double bytesAfterHaving = rowWidth * rowsAfterHaving; int parallelLevel = CostUtil.estimateParallelLevel( false, context.getConnection().getQueryServices()); Cost cost = CostUtil.estimateAggregateCost( inputBytes, bytesBeforeHaving, groupBy, parallelLevel); if (!orderBy.getOrderByExpressions().isEmpty()) { Cost orderByCost = CostUtil.estimateOrderByCost( bytesAfterHaving, outputBytes, parallelLevel); cost = cost.plus(orderByCost); } return super.getCost().plus(cost); }
@Override public final ResultIterator iterator(ParallelScanGrouper scanGrouper, Scan scan) throws SQLException { if (LOG.isDebugEnabled()) { LOG.debug(LogUtil.addCustomAnnotations("Gets ready for iteration: " + scan, connection)); } ResultIterator iterator = new GettingResultIterator(context, where); if (!orderBy.getOrderByExpressions().isEmpty()) { // TopN int thresholdBytes = context.getConnection().getQueryServices().getProps().getInt( QueryServices.SPOOL_THRESHOLD_BYTES_ATTRIB, QueryServicesOptions.DEFAULT_SPOOL_THRESHOLD_BYTES); iterator = new OrderedResultIterator(iterator, orderBy.getOrderByExpressions(), thresholdBytes, limit, offset, projector.getEstimatedRowByteSize()); } else { if (offset != null) { iterator = new OffsetResultIterator(iterator, offset); } if (limit != null) { iterator = new LimitingResultIterator(iterator, limit); } } if (context.getSequenceManager().getSequenceCount() > 0) { iterator = new SequenceResultIterator(iterator, context.getSequenceManager()); } if (LOG.isDebugEnabled()) { LOG.debug(LogUtil.addCustomAnnotations("Iterator ready: " + iterator, connection)); } TraceScope scope = Tracing.startNewSpan(context.getConnection(), "Creating basic query for " + getPlanSteps(iterator)); return (scope.getSpan() != null) ? new TracingIterator(scope, iterator) : iterator; }
@Override public ExplainPlan getExplainPlan() throws SQLException { List<String> planSteps = Lists.newArrayList(delegate.getExplainPlan().getPlanSteps()); if (where != null) { planSteps.add("CLIENT FILTER BY " + where.toString()); } if (!orderBy.getOrderByExpressions().isEmpty()) { if (offset != null) { planSteps.add("CLIENT OFFSET " + offset); } planSteps.add("CLIENT" + (limit == null ? "" : " TOP " + limit + " ROW" + (limit == 1 ? "" : "S")) + " SORTED BY " + orderBy.getOrderByExpressions().toString()); } else { if (offset != null) { planSteps.add("CLIENT OFFSET " + offset); } if (limit != null) { planSteps.add("CLIENT " + limit + " ROW LIMIT"); } } if (context.getSequenceManager().getSequenceCount() > 0) { int nSequences = context.getSequenceManager().getSequenceCount(); planSteps.add("CLIENT RESERVE VALUES FROM " + nSequences + " SEQUENCE" + (nSequences == 1 ? "" : "S")); } return new ExplainPlan(planSteps); }
@Override public Cost getCost() { Double outputBytes = this.accept(new ByteCountVisitor()); Double inputRows = this.getDelegate().accept(new RowCountVisitor()); Double rowWidth = this.accept(new AvgRowWidthVisitor()); if (inputRows == null || outputBytes == null || rowWidth == null) { return Cost.UNKNOWN; } double inputBytes = inputRows * rowWidth; double rowsBeforeHaving = RowCountVisitor.aggregate( RowCountVisitor.filter( inputRows.doubleValue(), RowCountVisitor.stripSkipScanFilter( context.getScan().getFilter())), groupBy); double rowsAfterHaving = RowCountVisitor.filter(rowsBeforeHaving, having); double bytesBeforeHaving = rowWidth * rowsBeforeHaving; double bytesAfterHaving = rowWidth * rowsAfterHaving; int parallelLevel = CostUtil.estimateParallelLevel( false, context.getConnection().getQueryServices()); Cost cost = CostUtil.estimateAggregateCost( inputBytes, bytesBeforeHaving, groupBy, parallelLevel); if (!orderBy.getOrderByExpressions().isEmpty()) { Cost orderByCost = CostUtil.estimateOrderByCost( bytesAfterHaving, outputBytes, parallelLevel); cost = cost.plus(orderByCost); } return super.getCost().plus(cost); }
/** * The method validates the statement passed to the query plan. List of conditions are * <ol> * <li>Is a SELECT statement</li> * <li>doesn't contain ORDER BY expression</li> * <li>doesn't contain LIMIT</li> * <li>doesn't contain GROUP BY expression</li> * <li>doesn't contain DISTINCT</li> * <li>doesn't contain AGGREGATE functions</li> * </ol> * @param queryPlan * @return */ private boolean isValidStatement(final QueryPlan queryPlan) { if(queryPlan.getStatement().getOperation() != PhoenixStatement.Operation.QUERY) { throw new IllegalArgumentException("Query passed isn't a SELECT statement"); } if(!queryPlan.getOrderBy().getOrderByExpressions().isEmpty() || queryPlan.getLimit() != null || (queryPlan.getGroupBy() != null && !queryPlan.getGroupBy().isEmpty()) || queryPlan.getStatement().isDistinct() || queryPlan.getStatement().isAggregate()) { throw new IllegalArgumentException("SELECT statement shouldn't contain DISTINCT or ORDER BY or LIMIT or GROUP BY expressions"); } return true; }
@Override public ResultIterator iterator(ParallelScanGrouper scanGrouper, Scan scan) throws SQLException { ResultIterator iterator = delegate.iterator(scanGrouper, scan); if (where != null) { iterator = new FilterResultIterator(iterator, where); } if (!orderBy.getOrderByExpressions().isEmpty()) { // TopN int thresholdBytes = context.getConnection().getQueryServices().getProps().getInt( QueryServices.SPOOL_THRESHOLD_BYTES_ATTRIB, QueryServicesOptions.DEFAULT_SPOOL_THRESHOLD_BYTES); iterator = new OrderedResultIterator(iterator, orderBy.getOrderByExpressions(), thresholdBytes, limit, offset, projector.getEstimatedRowByteSize()); } else { if (offset != null) { iterator = new OffsetResultIterator(iterator, offset); } if (limit != null) { iterator = new LimitingResultIterator(iterator, limit); } } if (context.getSequenceManager().getSequenceCount() > 0) { iterator = new SequenceResultIterator(iterator, context.getSequenceManager()); } return iterator; }
private ScanPlan(StatementContext context, FilterableStatement statement, TableRef table, RowProjector projector, Integer limit, Integer offset, OrderBy orderBy, ParallelIteratorFactory parallelIteratorFactory, boolean allowPageFilter, Expression dynamicFilter, QueryPlan dataPlan) throws SQLException { super(context, statement, table, projector, context.getBindManager().getParameterMetaData(), limit,offset, orderBy, GroupBy.EMPTY_GROUP_BY, parallelIteratorFactory != null ? parallelIteratorFactory : buildResultIteratorFactory(context, statement, table, orderBy, limit, offset, allowPageFilter), dynamicFilter, dataPlan); this.allowPageFilter = allowPageFilter; boolean isOrdered = !orderBy.getOrderByExpressions().isEmpty(); if (isOrdered) { // TopN int thresholdBytes = context.getConnection().getQueryServices().getProps().getInt( QueryServices.SPOOL_THRESHOLD_BYTES_ATTRIB, QueryServicesOptions.DEFAULT_SPOOL_THRESHOLD_BYTES); ScanRegionObserver.serializeIntoScan(context.getScan(), thresholdBytes, limit == null ? -1 : QueryUtil.getOffsetLimit(limit, offset), orderBy.getOrderByExpressions(), projector.getEstimatedRowByteSize()); } Integer perScanLimit = !allowPageFilter || isOrdered ? null : limit; perScanLimit = QueryUtil.getOffsetLimit(perScanLimit, offset); Pair<Long, Long> estimate = getEstimateOfDataSizeToScanIfWithinThreshold(context, table.getTable(), perScanLimit); this.isDataToScanWithinThreshold = estimate != null; this.isSerial = isSerial(context, statement, tableRef, orderBy, isDataToScanWithinThreshold); if (isSerial) { serialBytesEstimate = estimate.getFirst(); serialRowsEstimate = estimate.getSecond(); serialEstimateInfoTs = StatisticsUtil.NOT_STATS_BASED_TS; } }
@Override public ResultIterator iterator(ParallelScanGrouper scanGrouper, Scan scan) throws SQLException { ResultIterator iterator = delegate.iterator(scanGrouper, scan); if (where != null) { iterator = new FilterResultIterator(iterator, where); } if (!orderBy.getOrderByExpressions().isEmpty()) { // TopN int thresholdBytes = context.getConnection().getQueryServices().getProps().getInt( QueryServices.SPOOL_THRESHOLD_BYTES_ATTRIB, QueryServicesOptions.DEFAULT_SPOOL_THRESHOLD_BYTES); iterator = new OrderedResultIterator(iterator, orderBy.getOrderByExpressions(), thresholdBytes, limit, offset, projector.getEstimatedRowByteSize()); } else { if (offset != null) { iterator = new OffsetResultIterator(iterator, offset); } if (limit != null) { iterator = new LimitingResultIterator(iterator, limit); } } if (context.getSequenceManager().getSequenceCount() > 0) { iterator = new SequenceResultIterator(iterator, context.getSequenceManager()); } return iterator; }
@Override public ExplainPlan getExplainPlan() throws SQLException { List<String> planSteps = Lists.newArrayList(delegate.getExplainPlan().getPlanSteps()); if (where != null) { planSteps.add("CLIENT FILTER BY " + where.toString()); } if (!orderBy.getOrderByExpressions().isEmpty()) { if (offset != null) { planSteps.add("CLIENT OFFSET " + offset); } planSteps.add("CLIENT" + (limit == null ? "" : " TOP " + limit + " ROW" + (limit == 1 ? "" : "S")) + " SORTED BY " + orderBy.getOrderByExpressions().toString()); } else { if (offset != null) { planSteps.add("CLIENT OFFSET " + offset); } if (limit != null) { planSteps.add("CLIENT " + limit + " ROW LIMIT"); } } if (context.getSequenceManager().getSequenceCount() > 0) { int nSequences = context.getSequenceManager().getSequenceCount(); planSteps.add("CLIENT RESERVE VALUES FROM " + nSequences + " SEQUENCE" + (nSequences == 1 ? "" : "S")); } return new ExplainPlan(planSteps); }
private ScanPlan(StatementContext context, FilterableStatement statement, TableRef table, RowProjector projector, Integer limit, Integer offset, OrderBy orderBy, ParallelIteratorFactory parallelIteratorFactory, boolean allowPageFilter, Expression dynamicFilter, QueryPlan dataPlan) throws SQLException { super(context, statement, table, projector, context.getBindManager().getParameterMetaData(), limit,offset, orderBy, GroupBy.EMPTY_GROUP_BY, parallelIteratorFactory != null ? parallelIteratorFactory : buildResultIteratorFactory(context, statement, table, orderBy, limit, offset, allowPageFilter), dynamicFilter, dataPlan); this.allowPageFilter = allowPageFilter; boolean isOrdered = !orderBy.getOrderByExpressions().isEmpty(); if (isOrdered) { // TopN int thresholdBytes = context.getConnection().getQueryServices().getProps().getInt( QueryServices.SPOOL_THRESHOLD_BYTES_ATTRIB, QueryServicesOptions.DEFAULT_SPOOL_THRESHOLD_BYTES); ScanRegionObserver.serializeIntoScan(context.getScan(), thresholdBytes, limit == null ? -1 : QueryUtil.getOffsetLimit(limit, offset), orderBy.getOrderByExpressions(), projector.getEstimatedRowByteSize()); } Integer perScanLimit = !allowPageFilter || isOrdered ? null : limit; perScanLimit = QueryUtil.getOffsetLimit(perScanLimit, offset); Pair<Long, Long> estimate = getEstimateOfDataSizeToScanIfWithinThreshold(context, table.getTable(), perScanLimit); this.isDataToScanWithinThreshold = estimate != null; this.isSerial = isSerial(context, statement, tableRef, orderBy, isDataToScanWithinThreshold); if (isSerial) { serialBytesEstimate = estimate.getFirst(); serialRowsEstimate = estimate.getSecond(); serialEstimateInfoTs = StatisticsUtil.NOT_STATS_BASED_TS; } }
@Override public Cost getCost() { Long byteCount = null; try { byteCount = getEstimatedBytesToScan(); } catch (SQLException e) { // ignored. } Double outputBytes = this.accept(new ByteCountVisitor()); if (byteCount == null || outputBytes == null) { return Cost.UNKNOWN; } int parallelLevel = CostUtil.estimateParallelLevel( true, context.getConnection().getQueryServices()); Cost cost = new Cost(0, 0, byteCount); if (!orderBy.getOrderByExpressions().isEmpty()) { Cost orderByCost = CostUtil.estimateOrderByCost(byteCount, outputBytes, parallelLevel); cost = cost.plus(orderByCost); } return cost; }
@Override public Cost getCost() { Long byteCount = null; try { byteCount = getEstimatedBytesToScan(); } catch (SQLException e) { // ignored. } Double outputBytes = this.accept(new ByteCountVisitor()); if (byteCount == null || outputBytes == null) { return Cost.UNKNOWN; } int parallelLevel = CostUtil.estimateParallelLevel( true, context.getConnection().getQueryServices()); Cost cost = new Cost(0, 0, byteCount); if (!orderBy.getOrderByExpressions().isEmpty()) { Cost orderByCost = CostUtil.estimateOrderByCost(byteCount, outputBytes, parallelLevel); cost = cost.plus(orderByCost); } return cost; }
@SuppressWarnings("deprecation") private static ParallelIteratorFactory buildResultIteratorFactory(StatementContext context, FilterableStatement statement, TableRef tableRef, OrderBy orderBy, Integer limit,Integer offset, boolean allowPageFilter) throws SQLException { if ((isSerial(context, statement, tableRef, orderBy, getEstimateOfDataSizeToScanIfWithinThreshold(context, tableRef.getTable(), QueryUtil.getOffsetLimit(limit, offset)) != null) || isRoundRobinPossible(orderBy, context) || isPacingScannersPossible(context))) { return ParallelIteratorFactory.NOOP_FACTORY; } ParallelIteratorFactory spoolingResultIteratorFactory = new SpoolingResultIterator.SpoolingResultIteratorFactory( context.getConnection().getQueryServices()); // If we're doing an order by then we need the full result before we can do anything, // so we don't bother chunking it. If we're just doing a simple scan then we chunk // the scan to have a quicker initial response. if (!orderBy.getOrderByExpressions().isEmpty()) { return spoolingResultIteratorFactory; } else { return new ChunkedResultIterator.ChunkedResultIteratorFactory( spoolingResultIteratorFactory, context.getConnection().getMutationState(), tableRef); } }
@SuppressWarnings("deprecation") private static ParallelIteratorFactory buildResultIteratorFactory(StatementContext context, FilterableStatement statement, TableRef tableRef, OrderBy orderBy, Integer limit,Integer offset, boolean allowPageFilter) throws SQLException { if ((isSerial(context, statement, tableRef, orderBy, getEstimateOfDataSizeToScanIfWithinThreshold(context, tableRef.getTable(), QueryUtil.getOffsetLimit(limit, offset)) != null) || isRoundRobinPossible(orderBy, context) || isPacingScannersPossible(context))) { return ParallelIteratorFactory.NOOP_FACTORY; } ParallelIteratorFactory spoolingResultIteratorFactory = new SpoolingResultIterator.SpoolingResultIteratorFactory( context.getConnection().getQueryServices()); // If we're doing an order by then we need the full result before we can do anything, // so we don't bother chunking it. If we're just doing a simple scan then we chunk // the scan to have a quicker initial response. if (!orderBy.getOrderByExpressions().isEmpty()) { return spoolingResultIteratorFactory; } else { return new ChunkedResultIterator.ChunkedResultIteratorFactory( spoolingResultIteratorFactory, context.getConnection().getMutationState(), tableRef); } }
@Override public final ResultIterator iterator(ParallelScanGrouper scanGrouper, Scan scan) throws SQLException { this.iterators = new UnionResultIterators(plans, parentContext); ResultIterator scanner; boolean isOrdered = !orderBy.getOrderByExpressions().isEmpty(); if (isOrdered) { // TopN scanner = new MergeSortTopNResultIterator(iterators, limit, offset, orderBy.getOrderByExpressions()); } else { scanner = new ConcatResultIterator(iterators); if (offset != null) { scanner = new OffsetResultIterator(scanner, offset); } if (limit != null) { scanner = new LimitingResultIterator(scanner, limit); } } return scanner; }
@Override public Cost getCost() { Double inputBytes = this.getDelegate().accept(new ByteCountVisitor()); Double outputBytes = this.accept(new ByteCountVisitor()); if (inputBytes == null || outputBytes == null) { return Cost.UNKNOWN; } int parallelLevel = CostUtil.estimateParallelLevel( false, context.getConnection().getQueryServices()); Cost cost = new Cost(0, 0, 0); if (!orderBy.getOrderByExpressions().isEmpty()) { Cost orderByCost = CostUtil.estimateOrderByCost(inputBytes, outputBytes, parallelLevel); cost = cost.plus(orderByCost); } return super.getCost().plus(cost); }
@Override public final ResultIterator iterator(ParallelScanGrouper scanGrouper, Scan scan) throws SQLException { this.iterators = new UnionResultIterators(plans, parentContext); ResultIterator scanner; boolean isOrdered = !orderBy.getOrderByExpressions().isEmpty(); if (isOrdered) { // TopN scanner = new MergeSortTopNResultIterator(iterators, limit, offset, orderBy.getOrderByExpressions()); } else { scanner = new ConcatResultIterator(iterators); if (offset != null) { scanner = new OffsetResultIterator(scanner, offset); } if (limit != null) { scanner = new LimitingResultIterator(scanner, limit); } } return scanner; }
@Override public Cost getCost() { Double inputBytes = this.getDelegate().accept(new ByteCountVisitor()); Double outputBytes = this.accept(new ByteCountVisitor()); if (inputBytes == null || outputBytes == null) { return Cost.UNKNOWN; } int parallelLevel = CostUtil.estimateParallelLevel( false, context.getConnection().getQueryServices()); Cost cost = new Cost(0, 0, 0); if (!orderBy.getOrderByExpressions().isEmpty()) { Cost orderByCost = CostUtil.estimateOrderByCost(inputBytes, outputBytes, parallelLevel); cost = cost.plus(orderByCost); } return super.getCost().plus(cost); }
public static final boolean canQueryBeExecutedSerially(PTable table, OrderBy orderBy, StatementContext context) { /* * If ordering by columns not on the PK axis, we can't execute a query serially because we * need to do a merge sort across all the scans which isn't possible with SerialIterators. * Similar reasoning follows for salted and local index tables when ordering rows in a row * key order. Serial execution is OK in other cases since SerialIterators will execute scans * in the correct order. */ if (!orderBy.getOrderByExpressions().isEmpty() || ((table.getBucketNum() != null || table.getIndexType() == IndexType.LOCAL) && shouldRowsBeInRowKeyOrder( orderBy, context))) { return false; } return true; }