private List<RexNode> extractFilterPreds(Filter filterOp) { List<RexNode> conjs = new ArrayList<>(); for (RexNode r : HiveRelOptUtil.conjunctions(filterOp.getCondition())) { if (r.getKind() == SqlKind.IS_NOT_NULL) { RexCall isNotNullNode = (RexCall) r; if (RexUtil.isReferenceOrAccess(isNotNullNode.getOperands().get(0), true)) { ImmutableBitSet ref = RelOptUtil.InputFinder.bits(isNotNullNode); RelColumnOrigin co = mq.getColumnOrigin(filterOp, ref.nextSetBit(0)); if (co == null) { // We add it back conjs.add(r); continue; } RelOptHiveTable table = (RelOptHiveTable) co.getOriginTable(); List<ColStatistics> colStats = table.getColStat(ImmutableList.of(co.getOriginColumnOrdinal()), true); if (colStats == null || colStats.isEmpty() || colStats.get(0).getNumNulls() != 0) { // We add it back conjs.add(r); } } } else { conjs.add(r); } } return conjs; }
final ImmutableBitSet aggregateProjectBits = RelOptUtil.InputFinder.bits(aggregateProject.getChildExps(), null); for (int i = dimensions.size() - 1; i >= 0; i--) { final DimensionExpression dimension = dimensions.get(i);
private void infer(List<RexNode> predicates, Set<String> allExprsDigests, List<RexNode> inferedPredicates, List<RexNode> nonFieldsPredicates, boolean includeEqualityInference, ImmutableBitSet inferringFields) { for (RexNode r : predicates) { if (!includeEqualityInference && equalityPredicates.contains(r.toString())) { continue; } Iterable<Mapping> ms = mappings(r); if (ms.iterator().hasNext()) { for (Mapping m : ms) { RexNode tr = r.accept( new RexPermuteInputsShuttle(m, joinRel.getInput(0), joinRel.getInput(1))); if (inferringFields.contains(RelOptUtil.InputFinder.bits(tr)) && !allExprsDigests.contains(tr.toString()) && !isAlwaysTrue(tr)) { inferedPredicates.add(tr); allExprsDigests.add(tr.toString()); } } } else { if (!isAlwaysTrue(r)) { nonFieldsPredicates.add(r); } } } }
ImmutableBitSet rCols = RelOptUtil.InputFinder.bits(r); if (!rCols.isEmpty() && groupKeys.contains(rCols)) { r = r.accept(new RexPermuteInputsShuttle(m, input));
private void infer(List<RexNode> predicates, Set<String> allExprsDigests, List<RexNode> inferedPredicates, List<RexNode> nonFieldsPredicates, boolean includeEqualityInference, ImmutableBitSet inferringFields) { for (RexNode r : predicates) { if (!includeEqualityInference && equalityPredicates.contains(r.toString())) { continue; } Iterable<Mapping> ms = mappings(r); if (ms.iterator().hasNext()) { for (Mapping m : ms) { RexNode tr = r.accept( new RexPermuteInputsShuttle(m, joinRel.getInput(0), joinRel.getInput(1))); if (inferringFields.contains(RelOptUtil.InputFinder.bits(tr)) && !allExprsDigests.contains(tr.toString()) && !isAlwaysTrue(tr)) { inferedPredicates.add(tr); allExprsDigests.add(tr.toString()); } } } else { if (!isAlwaysTrue(r)) { nonFieldsPredicates.add(r); } } } }
RelOptUtil.InputFinder.bits(project.getProjects(), null); final ImmutableBitSet rightBits = ImmutableBitSet.range(
ImmutableBitSet rCols = RelOptUtil.InputFinder.bits(r); if (!rCols.isEmpty() && groupKeys.contains(rCols)) { r = r.accept(new RexPermuteInputsShuttle(m, input));
final Aggregate aggregate = call.rel(1); final ImmutableBitSet projectBits = RelOptUtil.InputFinder.bits(project.getChildExps(), null);
@Override public void visit(RelNode node, int ordinal, RelNode parent) { if (node instanceof HepRelVertex) { node = ((HepRelVertex) node).getCurrentRel(); } if (node instanceof TableScan) { simpleTree = true; } else if (node instanceof Project) { simpleTree = isSimple((Project) node); nonFilteringTree &= simpleTree; } else if (node instanceof Filter) { // Remove is not null from condition if it does not filter anything. Filter filterOp = (Filter) node; List<RexNode> conjs = extractFilterPreds(filterOp); ImmutableBitSet condBits = RelOptUtil.InputFinder.bits(conjs, null); // For simple tree, we want to know whether filter is only on // key columns. simpleTree = isSimple(condBits, filterOp, mq); nonFilteringTree &= conjs.isEmpty(); } else { simpleTree = false; nonFilteringTree = false; } if (simpleTree) { super.visit(node, ordinal, parent); } }
public void computePartitionList(HiveConf conf, RexNode pruneNode, Set<Integer> partOrVirtualCols) { try { if (!hiveTblMetadata.isPartitioned() || pruneNode == null || InputFinder.bits(pruneNode).length() == 0) { // there is no predicate on partitioning column, we need all partitions // in this case. partitionList = PartitionPruner.prune(hiveTblMetadata, null, conf, getName(), partitionCache); return; } // We have valid pruning expressions, only retrieve qualifying partitions ExprNodeDesc pruneExpr = pruneNode.accept(new ExprNodeConverter(getName(), getRowType(), partOrVirtualCols, this.getRelOptSchema().getTypeFactory())); partitionList = PartitionPruner.prune(hiveTblMetadata, pruneExpr, conf, getName(), partitionCache); } catch (HiveException he) { throw new RuntimeException(he); } }
public void computePartitionList(HiveConf conf, RexNode pruneNode, Set<Integer> partOrVirtualCols) { try { if (!hiveTblMetadata.isPartitioned() || pruneNode == null || InputFinder.bits(pruneNode).length() == 0) { // there is no predicate on partitioning column, we need all partitions // in this case. partitionList = PartitionPruner.prune(hiveTblMetadata, null, conf, getName(), partitionCache); return; } // We have valid pruning expressions, only retrieve qualifying partitions ExprNodeDesc pruneExpr = pruneNode.accept(new ExprNodeConverter(getName(), getRowType(), partOrVirtualCols, getTypeFactory())); partitionList = PartitionPruner.prune(hiveTblMetadata, pruneExpr, conf, getName(), partitionCache); } catch (HiveException he) { throw new RuntimeException(he); } }
private boolean filterRefersToBothSidesOfJoin(RexNode filter, Join j) { boolean refersToBothSides = false; int joinNoOfProjects = j.getRowType().getFieldCount(); ImmutableBitSet filterProjs = ImmutableBitSet.FROM_BIT_SET.apply(new BitSet(joinNoOfProjects)); ImmutableBitSet allLeftProjs = filterProjs.union(ImmutableBitSet.range(0, j.getInput(0) .getRowType().getFieldCount())); ImmutableBitSet allRightProjs = filterProjs.union(ImmutableBitSet.range(j.getInput(0) .getRowType().getFieldCount(), joinNoOfProjects)); filterProjs = filterProjs.union(InputFinder.bits(filter)); if (allLeftProjs.intersects(filterProjs) && allRightProjs.intersects(filterProjs)) refersToBothSides = true; return refersToBothSides; } }
private boolean isSimple(Filter filter, RelMetadataQuery mq) { ImmutableBitSet condBits = RelOptUtil.InputFinder.bits(filter.getCondition()); return isKey(condBits, filter, mq); }
private boolean isPartitionPredicate(RexNode expr, RelNode r) { if (r instanceof Project) { expr = RelOptUtil.pushFilterPastProject(expr, (Project) r); return isPartitionPredicate(expr, ((Project) r).getInput()); } else if (r instanceof Filter) { return isPartitionPredicate(expr, ((Filter) r).getInput()); } else if (r instanceof HiveTableScan) { RelOptHiveTable table = (RelOptHiveTable) ((HiveTableScan) r).getTable(); ImmutableBitSet cols = RelOptUtil.InputFinder.bits(expr); return table.containsPartitionColumnsOnly(cols); } return false; }
public void onMatch(RelOptRuleCall call) { final Project topProject = call.rel(0); final Project bottomProject = call.rel(1); // If top project does not reference any column at the bottom project, // we can just remove botton project final ImmutableBitSet topRefs = RelOptUtil.InputFinder.bits(topProject.getChildExps(), null); if (topRefs.isEmpty()) { RelBuilder relBuilder = call.builder(); relBuilder.push(bottomProject.getInput()); relBuilder.project(topProject.getChildExps()); call.transformTo(relBuilder.build()); return; } super.onMatch(call); }
private boolean filterRefersToBothSidesOfJoin(RexNode filter, Join j) { boolean refersToBothSides = false; int joinNoOfProjects = j.getRowType().getFieldCount(); ImmutableBitSet filterProjs = ImmutableBitSet.FROM_BIT_SET.apply(new BitSet(joinNoOfProjects)); ImmutableBitSet allLeftProjs = filterProjs.union(ImmutableBitSet.range(0, j.getInput(0) .getRowType().getFieldCount())); ImmutableBitSet allRightProjs = filterProjs.union(ImmutableBitSet.range(j.getInput(0) .getRowType().getFieldCount(), joinNoOfProjects)); filterProjs = filterProjs.union(InputFinder.bits(filter)); if (allLeftProjs.intersects(filterProjs) && allRightProjs.intersects(filterProjs)) refersToBothSides = true; return refersToBothSides; } }
private boolean isPartitionPredicate(RexNode expr, RelNode r) { if (r instanceof Project) { expr = RelOptUtil.pushFilterPastProject(expr, (Project) r); return isPartitionPredicate(expr, ((Project) r).getInput()); } else if (r instanceof Filter) { return isPartitionPredicate(expr, ((Filter) r).getInput()); } else if (r instanceof HiveTableScan) { RelOptHiveTable table = (RelOptHiveTable) ((HiveTableScan) r).getTable(); ImmutableBitSet cols = RelOptUtil.InputFinder.bits(expr); return table.containsPartitionColumnsOnly(cols); } return false; }
@Override public void onMatch(RelOptRuleCall call) { final Project project = call.rel(0); final Join join = call.rel(1); final RelNode left = call.rel(2); final Aggregate aggregate = call.rel(3); final ImmutableBitSet topRefs = RelOptUtil.InputFinder.bits(project.getChildExps(), null); perform(call, topRefs, project, join, left, aggregate); } }