private List<QueryPlan> getApplicablePlansForSingleFlatQuery(QueryPlan dataPlan, PhoenixStatement statement, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory, boolean stopAtBestPlan) throws SQLException { SelectStatement select = (SelectStatement)dataPlan.getStatement(); if (dataPlan.getContext().getScanRanges().isPointLookup() && stopAtBestPlan) { return Collections.<QueryPlan> singletonList(dataPlan); List<PTable>indexes = Lists.newArrayList(dataPlan.getTableRef().getTable().getIndexes()); if (indexes.isEmpty() || dataPlan.isDegenerate() || dataPlan.getTableRef().hasDynamicCols() || select.getHint().hasHint(Hint.NO_INDEX)) { return Collections.<QueryPlan> singletonList(dataPlan); List<? extends ColumnProjector> projectors = dataPlan.getProjector().getColumnProjectors(); List<PDatum> targetDatums = Lists.newArrayListWithExpectedSize(projectors.size()); for (ColumnProjector projector : projectors) { SelectStatement translatedIndexSelect = IndexStatementRewriter.translate(select, FromCompiler.getResolver(dataPlan.getTableRef())); List<QueryPlan> plans = Lists.newArrayListWithExpectedSize(1 + indexes.size()); plans.add(dataPlan); if (plan != null) { if (plan.isDegenerate()) { return Collections.singletonList(plan);
public UnionResultIterators(List<QueryPlan> plans, StatementContext parentStmtCtx) throws SQLException { this.parentStmtCtx = parentStmtCtx; int nPlans = plans.size(); iterators = Lists.newArrayListWithExpectedSize(nPlans); splits = Lists.newArrayListWithExpectedSize(nPlans * 30); scans = Lists.newArrayListWithExpectedSize(nPlans * 10); readMetricsList = Lists.newArrayListWithCapacity(nPlans); overAllQueryMetricsList = Lists.newArrayListWithCapacity(nPlans); for (QueryPlan plan : plans) { readMetricsList.add(plan.getContext().getReadMetricsQueue()); overAllQueryMetricsList.add(plan.getContext().getOverallQueryMetrics()); iterators.add(LookAheadResultIterator.wrap(plan.iterator())); splits.addAll(plan.getSplits()); scans.addAll(plan.getScans()); } }
@Override public boolean useRoundRobinIterator() throws SQLException { return delegate.useRoundRobinIterator(); }
private synchronized void openCursor(Connection conn) throws SQLException { if(isOpen){ return; } this.scan = this.queryPlan.getContext().getScan(); isReversed=OrderBy.REV_ROW_KEY_ORDER_BY.equals(this.queryPlan.getOrderBy()); isOpen = true; }
/** * The method validates the statement passed to the query plan. List of conditions are * <ol> * <li>Is a SELECT statement</li> * <li>doesn't contain ORDER BY expression</li> * <li>doesn't contain LIMIT</li> * <li>doesn't contain GROUP BY expression</li> * <li>doesn't contain DISTINCT</li> * <li>doesn't contain AGGREGATE functions</li> * </ol> * @param queryPlan * @return */ private boolean isValidStatement(final QueryPlan queryPlan) { if(queryPlan.getStatement().getOperation() != PhoenixStatement.Operation.QUERY) { throw new IllegalArgumentException("Query passed isn't a SELECT statement"); } if(!queryPlan.getOrderBy().getOrderByExpressions().isEmpty() || queryPlan.getLimit() != null || (queryPlan.getGroupBy() != null && !queryPlan.getGroupBy().isEmpty()) || queryPlan.getStatement().isDistinct() || queryPlan.getStatement().isAggregate()) { throw new IllegalArgumentException("SELECT statement shouldn't contain DISTINCT or ORDER BY or LIMIT or GROUP BY expressions"); } return true; }
@Test public void testNoLocalIndexPruning() throws SQLException { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); try (Connection conn = DriverManager.getConnection(getUrl(), props)) { conn.createStatement().execute("CREATE TABLE T (\n" + " A CHAR(1) NOT NULL,\n" + " B CHAR(1) NOT NULL,\n" + " C CHAR(1) NOT NULL,\n" + " CONSTRAINT PK PRIMARY KEY (\n" + " A,\n" + " B,\n" + " C\n" + " )\n" + ") SPLIT ON ('A','C','E','G','I')"); conn.createStatement().execute("CREATE LOCAL INDEX IDX ON T(C)"); String query = "SELECT * FROM T WHERE C='C'"; PhoenixStatement statement = conn.createStatement().unwrap(PhoenixStatement.class); QueryPlan plan = statement.optimizeQuery(query); assertEquals("IDX", plan.getContext().getCurrentTable().getTable().getName().getString()); plan.iterator(); List<List<Scan>> outerScans = plan.getScans(); assertEquals(6, outerScans.size()); } }
LOG.info("Generating iterators for " + scans.size() + " scans in keyrange: " + pSplit.getKeyRange()); List<PeekingResultIterator> iterators = Lists.newArrayListWithExpectedSize(scans.size()); StatementContext ctx = queryPlan.getContext(); ReadMetricQueue readMetrics = ctx.getReadMetricsQueue(); String tableName = queryPlan.getTableRef().getTable().getPhysicalName().getString(); String snapshotName = this.configuration.get(PhoenixConfigurationUtil.SNAPSHOT_NAME_KEY); byte[] tableNameBytes = queryPlan.getTableRef().getTable().getPhysicalName().getBytes(); ConnectionQueryServices services = queryPlan.getContext().getConnection().getQueryServices(); services.clearTableRegionCache(TableName.valueOf(tableNameBytes)); long renewScannerLeaseThreshold = queryPlan.getContext().getConnection().getQueryServices().getRenewLeaseThresholdMilliSeconds(); for (Scan scan : scans) { ScanMetricsHolder scanMetricsHolder = ScanMetricsHolder.getInstance(readMetrics, tableName, scan, queryPlan.getContext().getConnection().getLogLevel()); if (snapshotName != null) { final TableResultIterator tableResultIterator = new TableResultIterator( queryPlan.getContext().getConnection().getMutationState(), scan, scanMetricsHolder, renewScannerLeaseThreshold, queryPlan, MapReduceParallelScanGrouper.getInstance()); ResultIterator iterator = queryPlan.useRoundRobinIterator() ? RoundRobinResultIterator.newIterator(iterators, queryPlan) : ConcatResultIterator.newIterator(iterators); if(queryPlan.getContext().getSequenceManager().getSequenceCount() > 0) { iterator = new SequenceResultIterator(iterator, queryPlan.getContext().getSequenceManager());
public BaseResultIterators(QueryPlan plan, Integer perScanLimit, Integer offset, ParallelScanGrouper scanGrouper, Scan scan, Map<ImmutableBytesPtr,ServerCache> caches, QueryPlan dataPlan) throws SQLException { super(plan.getContext(), plan.getTableRef(), plan.getGroupBy(), plan.getOrderBy(), plan.getStatement().getHint(), QueryUtil.getOffsetLimit(plan.getLimit(), plan.getOffset()), offset); this.plan = plan; this.scan = scan; this.scanGrouper = scanGrouper; this.dataPlan = dataPlan; StatementContext context = plan.getContext(); TableRef tableRef = plan.getTableRef(); PTable table = tableRef.getTable(); physicalTableName = table.getPhysicalName().getBytes();
PhoenixStatement statement = conn.createStatement().unwrap(PhoenixStatement.class); QueryPlan plan = statement.optimizeQuery(query); List<QueryPlan> childPlans = plan.accept(new MultipleChildrenExtractor()); assertEquals(2, childPlans.size()); assertEquals("IDX1", childPlans.get(0).getContext().getCurrentTable().getTable().getName().getString()); childPlans.get(0).iterator(); List<List<Scan>> outerScansL = childPlans.get(0).getScans(); assertEquals(1, outerScansL.size()); List<Scan> innerScansL = outerScansL.get(0); assertEquals("C", Bytes.toString(scanL.getStopRow()).trim()); assertEquals("IDX2", childPlans.get(1).getContext().getCurrentTable().getTable().getName().getString()); childPlans.get(1).iterator(); List<List<Scan>> outerScansR = childPlans.get(1).getScans(); assertEquals(2, outerScansR.size()); List<Scan> innerScansR1 = outerScansR.get(0);
@Override public ResultIterator iterator(ParallelScanGrouper scanGrouper, Scan scan) throws SQLException { StatementContext context = delegate.getContext(); if (resultIterator == null) { context.getOverallQueryMetrics().startQuery(); resultIterator = new CursorResultIterator(LookAheadResultIterator.wrap(delegate.iterator(scanGrouper, scan)),cursorName); } return resultIterator; }
@Test public void testIgnoreIndexesBasedOnHint() throws Exception { Connection conn = DriverManager.getConnection(getUrl()); conn.createStatement().execute("CREATE TABLE t (k INTEGER NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) IMMUTABLE_ROWS=true"); conn.createStatement().execute("CREATE INDEX idx1 ON t(v1) INCLUDE(v2)"); conn.createStatement().execute("CREATE INDEX idx2 ON t(v1,v2)"); PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); QueryPlan plan = stmt.optimizeQuery("SELECT /*+NO_INDEX*/ k FROM t WHERE v1 = 'foo' AND v2 = 'bar'"); assertEquals("T", plan.getTableRef().getTable().getTableName().getString()); }
private static StatementContext compileStatement(String query, List<Object> binds, Integer limit) throws SQLException { PhoenixConnection pconn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)).unwrap(PhoenixConnection.class); PhoenixPreparedStatement pstmt = new PhoenixPreparedStatement(pconn, query); assertRoundtrip(query); TestUtil.bindParams(pstmt, binds); QueryPlan plan = pstmt.compileQuery(); assertEquals(limit, plan.getLimit()); return plan.getContext(); }
@Override public List<KeyRange> getSplits() { return delegate.getSplits(); }
public QueryPlan compileUnionAll(SelectStatement select) throws SQLException { List<SelectStatement> unionAllSelects = select.getSelects(); List<QueryPlan> plans = new ArrayList<QueryPlan>(); for (int i=0; i < unionAllSelects.size(); i++ ) { SelectStatement subSelect = unionAllSelects.get(i); // Push down order-by and limit into sub-selects. if (!select.getOrderBy().isEmpty() || select.getLimit() != null) { if (select.getOffset() == null) { subSelect = NODE_FACTORY.select(subSelect, select.getOrderBy(), select.getLimit(), null); } else { subSelect = NODE_FACTORY.select(subSelect, select.getOrderBy(), null, null); } } QueryPlan subPlan = compileSubquery(subSelect, true); plans.add(subPlan); } TableRef tableRef = UnionCompiler.contructSchemaTable(statement, plans, select.hasWildcard() ? null : select.getSelect()); ColumnResolver resolver = FromCompiler.getResolver(tableRef); StatementContext context = new StatementContext(statement, resolver, scan, sequenceManager); QueryPlan plan = compileSingleFlatQuery(context, select, statement.getParameters(), false, false, null, null, false); plan = new UnionPlan(context, select, tableRef, plan.getProjector(), plan.getLimit(), plan.getOffset(), plan.getOrderBy(), GroupBy.EMPTY_GROUP_BY, plans, context.getBindManager().getParameterMetaData()); return plan; }
protected QueryPlan compileSingleQuery(StatementContext context, SelectStatement select, List<Object> binds, boolean asSubquery, boolean allowPageFilter) throws SQLException{ SelectStatement innerSelect = select.getInnerSelectStatement(); if (innerSelect == null) { return compileSingleFlatQuery(context, select, binds, asSubquery, allowPageFilter, null, null, true); } QueryPlan innerPlan = compileSubquery(innerSelect, false); TupleProjector tupleProjector = new TupleProjector(innerPlan.getProjector()); innerPlan = new TupleProjectionPlan(innerPlan, tupleProjector, null); // Replace the original resolver and table with those having compiled type info. TableRef tableRef = context.getResolver().getTables().get(0); ColumnResolver resolver = FromCompiler.getResolverForCompiledDerivedTable(statement.getConnection(), tableRef, innerPlan.getProjector()); context.setResolver(resolver); tableRef = resolver.getTables().get(0); context.setCurrentTable(tableRef); boolean isInRowKeyOrder = innerPlan.getGroupBy() == GroupBy.EMPTY_GROUP_BY && innerPlan.getOrderBy() == OrderBy.EMPTY_ORDER_BY; return compileSingleFlatQuery(context, select, binds, asSubquery, allowPageFilter, innerPlan, tupleProjector, isInRowKeyOrder); }
@Override public ResultIterator iterator(ParallelScanGrouper scanGrouper, Scan scan) throws SQLException { return type == JoinType.Semi || type == JoinType.Anti ? new SemiAntiJoinIterator(lhsPlan.iterator(scanGrouper), rhsPlan.iterator(scanGrouper)) : new BasicJoinIterator(lhsPlan.iterator(scanGrouper), rhsPlan.iterator(scanGrouper)); }
@Override public Double visit(HashJoinPlan plan) { try { QueryPlan lhsPlan = plan.getDelegate(); Long b = lhsPlan.getEstimatedRowsToScan(); if (b == null) { return null; } Double rows = filter(b.doubleValue(), stripSkipScanFilter(lhsPlan.getContext().getScan().getFilter())); JoinTableNode.JoinType[] joinTypes = plan.getJoinInfo().getJoinTypes(); HashJoinPlan.SubPlan[] subPlans = plan.getSubPlans(); for (int i = 0; i < joinTypes.length; i++) { Double rhsRows = subPlans[i].getInnerPlan().accept(this); if (rhsRows == null) { return null; } rows = join(rows, rhsRows.doubleValue(), joinTypes[i]); } if (lhsPlan instanceof AggregatePlan) { AggregatePlan aggPlan = (AggregatePlan) lhsPlan; rows = filter(aggregate(rows, aggPlan.getGroupBy()), aggPlan.getHaving()); } return limit(rows, lhsPlan.getLimit()); } catch (SQLException e) { } return null; }
@Override public RowProjector getProjector() { return delegate.getProjector(); }
@Test public void testDistinctPrefixOnIntIndex() throws Exception { Connection conn = DriverManager.getConnection(getUrl()); conn.createStatement().execute("CREATE TABLE t (k INTEGER NOT NULL PRIMARY KEY, v1 INTEGER, v2 VARCHAR)"); conn.createStatement().execute("CREATE INDEX idx ON t(v1)"); PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); QueryPlan plan = stmt.optimizeQuery("SELECT COUNT(DISTINCT v1) FROM t"); assertTrue(plan.getGroupBy().isOrderPreserving()); assertFalse(plan.getGroupBy().getKeyExpressions().isEmpty()); assertEquals("IDX", plan.getTableRef().getTable().getTableName().getString()); }
final QueryPlan queryPlan = pstmt.compileQuery(selectStatement); isValidStatement(queryPlan); final String tableName = queryPlan.getTableRef().getTable().getName().getString(); final List<? extends ColumnProjector> projectedColumns = queryPlan.getProjector().getColumnProjectors(); final List<String> columns = Lists.transform(projectedColumns, new Function<ColumnProjector,String>() {