private static boolean mustPartition(JoinNode joinNode) { JoinNode.Type type = joinNode.getType(); // With REPLICATED, the unmatched rows from right-side would be duplicated. return type == RIGHT || type == FULL; }
@Override public boolean shapeMatches(PlanNode node) { if (!(node instanceof JoinNode)) { return false; } JoinNode joinNode = (JoinNode) node; return joinNode.getType() == joinType; }
private static PlanNode getOuterTable(JoinNode join) { checkState(join.getType() == JoinNode.Type.LEFT || join.getType() == JoinNode.Type.RIGHT, "expected LEFT or RIGHT JOIN"); PlanNode outerNode; if (join.getType().equals(JoinNode.Type.LEFT)) { outerNode = join.getLeft(); } else { outerNode = join.getRight(); } return outerNode; }
private static PlanNode getInnerTable(JoinNode join) { checkState(join.getType() == JoinNode.Type.LEFT || join.getType() == JoinNode.Type.RIGHT, "expected LEFT or RIGHT JOIN"); PlanNode innerNode; if (join.getType().equals(JoinNode.Type.LEFT)) { innerNode = join.getRight(); } else { innerNode = join.getLeft(); } return innerNode; }
JoinNodeFlattener(JoinNode node, Lookup lookup, int sourceLimit) { requireNonNull(node, "node is null"); checkState(node.getType() == INNER, "join type must be INNER"); this.outputSymbols = node.getOutputSymbols(); this.lookup = requireNonNull(lookup, "lookup is null"); flattenNode(node, sourceLimit); }
private static boolean mustReplicate(JoinNode joinNode, Context context) { JoinNode.Type type = joinNode.getType(); if (joinNode.getCriteria().isEmpty() && (type == INNER || type == LEFT)) { // There is nothing to partition on return true; } return isAtMostScalar(joinNode.getRight(), context.getLookup()); }
@Override public Void visitJoin(JoinNode node, Void context) { List<Expression> joinExpressions = new ArrayList<>(); for (JoinNode.EquiJoinClause clause : node.getCriteria()) { joinExpressions.add(clause.toExpression()); } String criteria = Joiner.on(" AND ").join(joinExpressions); printNode(node, node.getType().getJoinLabel(), criteria, NODE_COLORS.get(NodeType.JOIN)); node.getLeft().accept(this, context); node.getRight().accept(this, context); return null; }
@Override protected Optional<PlanNodeStatsEstimate> doCalculate(JoinNode node, StatsProvider sourceStats, Lookup lookup, Session session, TypeProvider types) { PlanNodeStatsEstimate leftStats = sourceStats.getStats(node.getLeft()); PlanNodeStatsEstimate rightStats = sourceStats.getStats(node.getRight()); PlanNodeStatsEstimate crossJoinStats = crossJoinStats(node, leftStats, rightStats, types); switch (node.getType()) { case INNER: return Optional.of(computeInnerJoinStats(node, crossJoinStats, session, types)); case LEFT: return Optional.of(computeLeftJoinStats(node, leftStats, rightStats, crossJoinStats, session, types)); case RIGHT: return Optional.of(computeRightJoinStats(node, leftStats, rightStats, crossJoinStats, session, types)); case FULL: return Optional.of(computeFullJoinStats(node, leftStats, rightStats, crossJoinStats, session, types)); default: throw new IllegalStateException("Unknown join type: " + node.getType()); } }
@Override public PhysicalOperation visitJoin(JoinNode node, LocalExecutionPlanContext context) { if (node.isCrossJoin()) { return createNestedLoopJoin(node, context); } List<JoinNode.EquiJoinClause> clauses = node.getCriteria(); List<Symbol> leftSymbols = Lists.transform(clauses, JoinNode.EquiJoinClause::getLeft); List<Symbol> rightSymbols = Lists.transform(clauses, JoinNode.EquiJoinClause::getRight); switch (node.getType()) { case INNER: case LEFT: case RIGHT: case FULL: return createLookupJoin(node, node.getLeft(), leftSymbols, node.getLeftHashSymbol(), node.getRight(), rightSymbols, node.getRightHashSymbol(), context); default: throw new UnsupportedOperationException("Unsupported join type: " + node.getType()); } }
private void flattenNode(PlanNode node, int limit) { PlanNode resolved = lookup.resolve(node); // (limit - 2) because you need to account for adding left and right side if (!(resolved instanceof JoinNode) || (sources.size() > (limit - 2))) { sources.add(node); return; } JoinNode joinNode = (JoinNode) resolved; if (joinNode.getType() != INNER || !isDeterministic(joinNode.getFilter().orElse(TRUE_LITERAL)) || joinNode.getDistributionType().isPresent()) { sources.add(node); return; } // we set the left limit to limit - 1 to account for the node on the right flattenNode(joinNode.getLeft(), limit - 1); flattenNode(joinNode.getRight(), limit); joinNode.getCriteria().stream() .map(EquiJoinClause::toExpression) .forEach(filters::add); joinNode.getFilter().ifPresent(filters::add); }
@Override protected Optional<PlanNode> pushDownProjectOff(PlanNodeIdAllocator idAllocator, JoinNode joinNode, Set<Symbol> referencedOutputs) { Optional<PlanNode> newLeft = restrictOutputs(idAllocator, joinNode.getLeft(), referencedOutputs); Optional<PlanNode> newRight = restrictOutputs(idAllocator, joinNode.getRight(), referencedOutputs); if (!newLeft.isPresent() && !newRight.isPresent()) { return Optional.empty(); } ImmutableList.Builder<Symbol> outputSymbolBuilder = ImmutableList.builder(); outputSymbolBuilder.addAll(newLeft.orElse(joinNode.getLeft()).getOutputSymbols()); outputSymbolBuilder.addAll(newRight.orElse(joinNode.getRight()).getOutputSymbols()); return Optional.of(new JoinNode( idAllocator.getNextId(), joinNode.getType(), newLeft.orElse(joinNode.getLeft()), newRight.orElse(joinNode.getRight()), joinNode.getCriteria(), outputSymbolBuilder.build(), joinNode.getFilter(), joinNode.getLeftHashSymbol(), joinNode.getRightHashSymbol(), joinNode.getDistributionType())); } }
@Override public JoinGraph visitJoin(JoinNode node, Context context) { //TODO: add support for non inner joins if (node.getType() != INNER) { return visitPlan(node, context); } JoinGraph left = node.getLeft().accept(this, context); JoinGraph right = node.getRight().accept(this, context); JoinGraph graph = left.joinWith(right, node.getCriteria(), context, node.getId()); if (node.getFilter().isPresent()) { return graph.withFilter(node.getFilter().get()); } return graph; }
@Override public Void visitJoin(JoinNode node, Integer indent) { List<Expression> joinExpressions = new ArrayList<>(); for (JoinNode.EquiJoinClause clause : node.getCriteria()) { joinExpressions.add(clause.toExpression()); } node.getFilter().ifPresent(joinExpressions::add); if (node.isCrossJoin()) { checkState(joinExpressions.isEmpty()); print(indent, "- CrossJoin => [%s]", formatOutputs(node.getOutputSymbols())); } else { print(indent, "- %s[%s]%s => [%s]", node.getType().getJoinLabel(), Joiner.on(" AND ").join(joinExpressions), formatHash(node.getLeftHashSymbol(), node.getRightHashSymbol()), formatOutputs(node.getOutputSymbols())); } node.getDistributionType().ifPresent(distributionType -> print(indent + 2, "Distribution: %s", distributionType)); node.getSortExpressionContext().ifPresent(context -> print(indent + 2, "SortExpression[%s]", context.getSortExpression())); printPlanNodesStatsAndCost(indent + 2, node); printStats(indent + 2, node.getId()); node.getLeft().accept(this, indent + 1); node.getRight().accept(this, indent + 1); return null; }
@Override public Void visitJoin(JoinNode node, Integer indent) { JoinNode.DistributionType distributionType = node.getDistributionType() .orElseThrow(() -> new VerifyException("Expected distribution type to be set")); if (node.isCrossJoin()) { checkState(node.getType() == INNER && distributionType == REPLICATED, "Expected CROSS JOIN to be INNER REPLICATED"); output(indent, "cross join:"); } else { output(indent, "join (%s, %s):", node.getType(), distributionType); } return visitPlan(node, indent + 1); }
@Override public PlanNode visitJoin(JoinNode node, RewriteContext<Void> context) { PlanNode left = context.rewrite(node.getLeft()); PlanNode right = context.rewrite(node.getRight()); List<JoinNode.EquiJoinClause> canonicalCriteria = canonicalizeJoinCriteria(node.getCriteria()); Optional<Expression> canonicalFilter = node.getFilter().map(this::canonicalize); Optional<Symbol> canonicalLeftHashSymbol = canonicalize(node.getLeftHashSymbol()); Optional<Symbol> canonicalRightHashSymbol = canonicalize(node.getRightHashSymbol()); if (node.getType().equals(INNER)) { canonicalCriteria.stream() .filter(clause -> types.get(clause.getLeft()).equals(types.get(clause.getRight()))) .filter(clause -> node.getOutputSymbols().contains(clause.getLeft())) .forEach(clause -> map(clause.getRight(), clause.getLeft())); } return new JoinNode(node.getId(), node.getType(), left, right, canonicalCriteria, canonicalizeAndDistinct(node.getOutputSymbols()), canonicalFilter, canonicalLeftHashSymbol, canonicalRightHashSymbol, node.getDistributionType()); }
private PlanWithProperties buildJoin(JoinNode node, PlanWithProperties newLeft, PlanWithProperties newRight, JoinNode.DistributionType newDistributionType) { JoinNode result = new JoinNode(node.getId(), node.getType(), newLeft.getNode(), newRight.getNode(), node.getCriteria(), node.getOutputSymbols(), node.getFilter(), node.getLeftHashSymbol(), node.getRightHashSymbol(), Optional.of(newDistributionType)); return new PlanWithProperties(result, deriveProperties(result, ImmutableList.of(newLeft.getProperties(), newRight.getProperties()))); }
private JoinNode tryNormalizeToOuterToInnerJoin(JoinNode node, Expression inheritedPredicate) { checkArgument(EnumSet.of(INNER, RIGHT, LEFT, FULL).contains(node.getType()), "Unsupported join type: %s", node.getType()); if (node.getType() == JoinNode.Type.INNER) { return node; } if (node.getType() == JoinNode.Type.FULL) { boolean canConvertToLeftJoin = canConvertOuterToInner(node.getLeft().getOutputSymbols(), inheritedPredicate); boolean canConvertToRightJoin = canConvertOuterToInner(node.getRight().getOutputSymbols(), inheritedPredicate); if (!canConvertToLeftJoin && !canConvertToRightJoin) { return node; } if (canConvertToLeftJoin && canConvertToRightJoin) { return new JoinNode(node.getId(), INNER, node.getLeft(), node.getRight(), node.getCriteria(), node.getOutputSymbols(), node.getFilter(), node.getLeftHashSymbol(), node.getRightHashSymbol(), node.getDistributionType()); } else { return new JoinNode(node.getId(), canConvertToLeftJoin ? LEFT : RIGHT, node.getLeft(), node.getRight(), node.getCriteria(), node.getOutputSymbols(), node.getFilter(), node.getLeftHashSymbol(), node.getRightHashSymbol(), node.getDistributionType()); } } if (node.getType() == JoinNode.Type.LEFT && !canConvertOuterToInner(node.getRight().getOutputSymbols(), inheritedPredicate) || node.getType() == JoinNode.Type.RIGHT && !canConvertOuterToInner(node.getLeft().getOutputSymbols(), inheritedPredicate)) { return node; } return new JoinNode(node.getId(), JoinNode.Type.INNER, node.getLeft(), node.getRight(), node.getCriteria(), node.getOutputSymbols(), node.getFilter(), node.getLeftHashSymbol(), node.getRightHashSymbol(), node.getDistributionType()); }
@Override protected Optional<PlanNode> pushDownProjectOff(PlanNodeIdAllocator idAllocator, JoinNode joinNode, Set<Symbol> referencedOutputs) { return Optional.of( new JoinNode( joinNode.getId(), joinNode.getType(), joinNode.getLeft(), joinNode.getRight(), joinNode.getCriteria(), filteredCopy(joinNode.getOutputSymbols(), referencedOutputs::contains), joinNode.getFilter(), joinNode.getLeftHashSymbol(), joinNode.getRightHashSymbol(), joinNode.getDistributionType())); } }
@Override public Result apply(JoinNode joinNode, Captures captures, Context context) { Optional<Expression> filter = joinNode.getFilter().map(x -> rewriter.rewrite(x, context)); if (!joinNode.getFilter().equals(filter)) { return Result.ofPlanNode(new JoinNode( joinNode.getId(), joinNode.getType(), joinNode.getLeft(), joinNode.getRight(), joinNode.getCriteria(), joinNode.getOutputSymbols(), filter, joinNode.getLeftHashSymbol(), joinNode.getRightHashSymbol(), joinNode.getDistributionType())); } return Result.empty(); } }
@Override public Result apply(AggregationNode aggregationNode, Captures captures, Context context) { JoinNode joinNode = captures.get(JOIN_NODE); if (joinNode.getType() != JoinNode.Type.INNER) { return Result.empty(); } // TODO: leave partial aggregation above Join? if (allAggregationsOn(aggregationNode.getAggregations(), joinNode.getLeft().getOutputSymbols())) { return Result.ofPlanNode(pushPartialToLeftChild(aggregationNode, joinNode, context)); } else if (allAggregationsOn(aggregationNode.getAggregations(), joinNode.getRight().getOutputSymbols())) { return Result.ofPlanNode(pushPartialToRightChild(aggregationNode, joinNode, context)); } return Result.empty(); }