Util.first(context.unwrap(RelFactories.AggregateFactory.class), HiveRelFactories.HIVE_AGGREGATE_FACTORY); this.filterFactory = Util.first(context.unwrap(RelFactories.FilterFactory.class), HiveRelFactories.HIVE_FILTER_FACTORY); this.projectFactory = Util.first(context.unwrap(RelFactories.ProjectFactory.class), HiveRelFactories.HIVE_PROJECT_FACTORY); this.sortFactory = Util.first(context.unwrap(RelFactories.SortFactory.class), HiveRelFactories.HIVE_SORT_FACTORY); this.setOpFactory = Util.first(context.unwrap(RelFactories.SetOpFactory.class), HiveRelFactories.HIVE_SET_OP_FACTORY); this.joinFactory = Util.first(context.unwrap(RelFactories.JoinFactory.class), HiveRelFactories.HIVE_JOIN_FACTORY); this.semiJoinFactory = Util.first(context.unwrap(RelFactories.SemiJoinFactory.class), HiveRelFactories.HIVE_SEMI_JOIN_FACTORY); this.correlateFactory = Util.first(context.unwrap(RelFactories.CorrelateFactory.class), RelFactories.DEFAULT_CORRELATE_FACTORY); this.valuesFactory = Util.first(context.unwrap(RelFactories.ValuesFactory.class), RelFactories.DEFAULT_VALUES_FACTORY); this.scanFactory = Util.first(context.unwrap(RelFactories.TableScanFactory.class), RelFactories.DEFAULT_TABLE_SCAN_FACTORY);
Util.first(context.unwrap(RelFactories.AggregateFactory.class), HiveRelFactories.HIVE_AGGREGATE_FACTORY); this.filterFactory = Util.first(context.unwrap(RelFactories.FilterFactory.class), HiveRelFactories.HIVE_FILTER_FACTORY); this.projectFactory = Util.first(context.unwrap(RelFactories.ProjectFactory.class), HiveRelFactories.HIVE_PROJECT_FACTORY); this.sortFactory = Util.first(context.unwrap(RelFactories.SortFactory.class), HiveRelFactories.HIVE_SORT_FACTORY); this.setOpFactory = Util.first(context.unwrap(RelFactories.SetOpFactory.class), HiveRelFactories.HIVE_SET_OP_FACTORY); this.joinFactory = Util.first(context.unwrap(RelFactories.JoinFactory.class), HiveRelFactories.HIVE_JOIN_FACTORY); this.semiJoinFactory = Util.first(context.unwrap(RelFactories.SemiJoinFactory.class), HiveRelFactories.HIVE_SEMI_JOIN_FACTORY); this.correlateFactory = Util.first(context.unwrap(RelFactories.CorrelateFactory.class), RelFactories.DEFAULT_CORRELATE_FACTORY); this.valuesFactory = Util.first(context.unwrap(RelFactories.ValuesFactory.class), RelFactories.DEFAULT_VALUES_FACTORY); this.scanFactory = Util.first(context.unwrap(RelFactories.TableScanFactory.class), RelFactories.DEFAULT_TABLE_SCAN_FACTORY);
/** * utility function to extract timezone id from Druid query * @param query Druid Rel * @return time zone */ private static TimeZone timezoneId(final DruidQuery query, final RexNode arg) { return arg.getType().getSqlTypeName() == SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE ? TimeZone.getTimeZone( query.getTopNode().getCluster().getPlanner().getContext().unwrap(CalciteConnectionConfig.class).timeZone()) : TimeZone.getTimeZone("UTC"); }
@Override public boolean matches(RelOptRuleCall call) { final Filter filter = call.rel(0); final RelNode filterChild = call.rel(1); // If the filter is already on top of a TableScan, // we can bail out if (filterChild instanceof TableScan) { return false; } HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class); // If this operator has been visited already by the rule, // we do not need to apply the optimization if (registry != null && registry.getVisited(this).contains(filter)) { return false; } return true; }
@Override public final HiveJoin copy(RelTraitSet traitSet, RexNode conditionExpr, RelNode left, RelNode right, JoinRelType joinType, boolean semiJoinDone) { try { Set<String> variablesStopped = Collections.emptySet(); HiveJoin join = new HiveJoin(getCluster(), traitSet, left, right, conditionExpr, joinType, variablesStopped, joinAlgorithm); // If available, copy state to registry for optimization rules HiveRulesRegistry registry = join.getCluster().getPlanner().getContext().unwrap(HiveRulesRegistry.class); if (registry != null) { registry.copyPushedPredicates(this, join); } return join; } catch (InvalidRelException | CalciteSemanticException e) { // Semantic error not possible. Must be a bug. Convert to // internal error. throw new AssertionError(e); } }
@Override public boolean matches(RelOptRuleCall call) { final Filter filter = call.rel(0); final RelNode filterChild = call.rel(1); // If the filter is already on top of a TableScan, // we can bail out if (filterChild instanceof TableScan) { return false; } HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class); // If this operator has been visited already by the rule, // we do not need to apply the optimization if (registry != null && registry.getVisited(this).contains(filter)) { return false; } return true; }
@Override public final HiveJoin copy(RelTraitSet traitSet, RexNode conditionExpr, RelNode left, RelNode right, JoinRelType joinType, boolean semiJoinDone) { try { Set<String> variablesStopped = Collections.emptySet(); HiveJoin join = new HiveJoin(getCluster(), traitSet, left, right, conditionExpr, joinType, variablesStopped, joinAlgorithm); // If available, copy state to registry for optimization rules HiveRulesRegistry registry = join.getCluster().getPlanner().getContext().unwrap(HiveRulesRegistry.class); if (registry != null) { registry.copyPushedPredicates(this, join); } return join; } catch (InvalidRelException | CalciteSemanticException e) { // Semantic error not possible. Must be a bug. Convert to // internal error. throw new AssertionError(e); } }
@Override public boolean isExecutable(HiveJoin join) { final Double maxMemory = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxMemory(); // Check streaming side RelNode smallInput = join.getStreamingInput(); if (smallInput == null) { return false; } return HiveAlgorithmsUtil.isFittingIntoMemory(maxMemory, smallInput, 1); }
public static Integer getSplitCountWithRepartition(HiveJoin join) { final Double maxSplitSize = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxSplitSize(); // We repartition: new number of splits final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); final Double averageRowSize = mq.getAverageRowSize(join); final Double rowCount = mq.getRowCount(join); if (averageRowSize == null || rowCount == null) { return null; } final Double totalSize = averageRowSize * rowCount; final Double splitCount = totalSize / maxSplitSize; return splitCount.intValue(); }
public static Integer getSplitCountWithRepartition(HiveJoin join) { final Double maxSplitSize = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxSplitSize(); // We repartition: new number of splits RelMetadataQuery mq = RelMetadataQuery.instance(); final Double averageRowSize = mq.getAverageRowSize(join); final Double rowCount = mq.getRowCount(join); if (averageRowSize == null || rowCount == null) { return null; } final Double totalSize = averageRowSize * rowCount; final Double splitCount = totalSize / maxSplitSize; return splitCount.intValue(); }
@Override public SemiJoin copy(RelTraitSet traitSet, RexNode condition, RelNode left, RelNode right, JoinRelType joinType, boolean semiJoinDone) { try { final JoinInfo joinInfo = JoinInfo.of(left, right, condition); HiveSemiJoin semijoin = new HiveSemiJoin(getCluster(), traitSet, left, right, condition, joinInfo.leftKeys, joinInfo.rightKeys); // If available, copy state to registry for optimization rules HiveRulesRegistry registry = semijoin.getCluster().getPlanner().getContext().unwrap(HiveRulesRegistry.class); if (registry != null) { registry.copyPushedPredicates(this, semijoin); } return semijoin; } catch (InvalidRelException | CalciteSemanticException e) { // Semantic error not possible. Must be a bug. Convert to // internal error. throw new AssertionError(e); } }
@Override public boolean isExecutable(HiveJoin join) { final Double maxMemory = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxMemory(); // Check streaming side RelNode smallInput = join.getStreamingInput(); if (smallInput == null) { return false; } return HiveAlgorithmsUtil.isFittingIntoMemory(maxMemory, smallInput, 1); }
@Override public SemiJoin copy(RelTraitSet traitSet, RexNode condition, RelNode left, RelNode right, JoinRelType joinType, boolean semiJoinDone) { try { final JoinInfo joinInfo = JoinInfo.of(left, right, condition); HiveSemiJoin semijoin = new HiveSemiJoin(getCluster(), traitSet, left, right, condition, joinInfo.leftKeys, joinInfo.rightKeys); // If available, copy state to registry for optimization rules HiveRulesRegistry registry = semijoin.getCluster().getPlanner().getContext().unwrap(HiveRulesRegistry.class); if (registry != null) { registry.copyPushedPredicates(this, semijoin); } return semijoin; } catch (InvalidRelException | CalciteSemanticException e) { // Semantic error not possible. Must be a bug. Convert to // internal error. throw new AssertionError(e); } }
@Override public boolean matches(RelOptRuleCall call) { final RelNode node = call.rel(0); numberMatches++; HiveRulesRegistry registry = call.getPlanner(). getContext().unwrap(HiveRulesRegistry.class); // If this operator has been visited already by the rule, // we do not need to apply the optimization if (registry != null && registry.getVisited(this).contains(node)) { return false; } return true; }
final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); final Double maxMemory = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxMemory();
if (noOfPE > 0) { boolean isCorrelatedColumns = j.getCluster().getPlanner().getContext(). unwrap(HiveConfPlannerContext.class).getIsCorrelatedColumns(); if (noOfPE > 1 && isCorrelatedColumns ){ ndvEstimate = maxNdvForCorrelatedColumns(peLst, colStatMap);
@Override public boolean isExecutable(HiveJoin join) { final Double maxMemory = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxMemory();
HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class); if (registry != null) { registry.registerVisited(this, filter);
HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class); if (registry != null) { registry.registerVisited(this, filter);
RelNode rChild = join.getRight(); HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class); assert registry != null;