public Integer splitCountRepartition(RelNode rel, RelMetadataQuery mq) { // We repartition: new number of splits final Double averageRowSize = mq.getAverageRowSize(rel); final Double rowCount = mq.getRowCount(rel); if (averageRowSize == null || rowCount == null) { return null; } final Double totalSize = averageRowSize * rowCount; final Double splitCount = totalSize / maxSplitSize; return splitCount.intValue(); }
public RelOptCost getCumulativeCost(HiveJoin rel, RelMetadataQuery mq) { RelOptCost cost = mq.getNonCumulativeCost(rel); List<RelNode> inputs = rel.getInputs(); RelOptCost maxICost = HiveCost.ZERO; for (RelNode input : inputs) { RelOptCost iCost = mq.getCumulativeCost(input); if (maxICost.isLt(iCost)) { maxICost = iCost; } } return cost.plus(maxICost); } }
for (RexNode pred : mq.getPulledUpPredicates(inp).pulledUpPredicates) { predicatesInSubtree.add(pred.toString()); predicatesInSubtree.addAll(Lists.transform(RelOptUtil.conjunctions(pred), REX_STR_FN));
protected FilterSelectivityEstimator(RelNode childRel) { super(true); this.childRel = childRel; this.childCardinality = RelMetadataQuery.instance().getRowCount(childRel); }
public static boolean isFittingIntoMemory(Double maxSize, RelNode input, int buckets) { Double currentMemory = RelMetadataQuery.instance().cumulativeMemoryWithinPhase(input); if (currentMemory != null) { if(currentMemory / buckets > maxSize) { return false; } return true; } return false; }
final double leftRowCount = mq.getRowCount(left); final double rightRowCount = mq.getRowCount(right); if (leftIsKey && rightIsKey) { if (rightRowCount < leftRowCount) { RexNode rightPred = RexUtil.composeConjunction( rexBuilder, rightFilters, true); double leftNDV = isPKSideSimpleTree ? mq.getDistinctRowCount(left, lBitSet, leftPred) : -1; double rightNDV = isPKSideSimpleTree ? mq.getDistinctRowCount(right, rBitSet, rightPred) : -1;
RelMetadataQuery mq = RelMetadataQuery.instance(); final ImmutableBitSet aggregateColumns = aggregate.getGroupSet(); final ImmutableBitSet keyColumns = keyColumns(aggregateColumns, mq.getPulledUpPredicates(join).pulledUpPredicates); final ImmutableBitSet joinColumns = RelOptUtil.InputFinder.bits(join.getCondition()); } else { final Boolean unique0 = mq.areColumnsUnique(joinInput, belowAggregateKey); unique = unique0 != null && unique0; RelOptCost afterCost = mq.getCumulativeCost(r); RelOptCost beforeCost = mq.getCumulativeCost(aggregate); if (afterCost.isLt(beforeCost)) { call.transformTo(r);
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { JoinAlgorithm oldAlgo = join.getJoinAlgorithm(); join.setJoinAlgorithm(TezCommonJoinAlgorithm.INSTANCE); final Double memoryWithinPhase = RelMetadataQuery.instance().cumulativeMemoryWithinPhase(join); final Integer splitCount = RelMetadataQuery.instance().splitCount(join); join.setJoinAlgorithm(oldAlgo); if (memoryWithinPhase == null || splitCount == null) { return null; } return memoryWithinPhase / splitCount; }
@Override public Double visitPrel(Prel prel, Void value) throws RuntimeException { RelMetadataQuery mq = RelMetadataQuery.instance(); return ((DrillCostBase) mq.getCumulativeCost(prel)).getMemory(); // return findCost(prel, mq); }
final RelMetadataQuery mq = call.getMetadataQuery(); final ImmutableBitSet keyColumns = keyColumns(aggregateColumns, mq.getPulledUpPredicates(join).pulledUpPredicates); final ImmutableBitSet joinColumns = RelOptUtil.InputFinder.bits(join.getCondition()); } else { final Boolean unique0 = mq.areColumnsUnique(joinInput, belowAggregateKey, true); unique = unique0 != null && unique0; RelOptCost afterCost = mq.getCumulativeCost(r); RelOptCost beforeCost = mq.getCumulativeCost(aggregate); if (afterCost.isLt(beforeCost)) { call.transformTo(r);
Set<RelColumnOrigin> colOrigs = RelMetadataQuery.instance().getColumnOrigins(aggregate, arg); if (null != colOrigs) { for (RelColumnOrigin colOrig : colOrigs) {
if ((nonDistinctCount == 0) && (argListSets.size() == 1)) { for (Integer arg : argListSets.iterator().next()) { Set<RelColumnOrigin> colOrigs = mq.getColumnOrigins(aggregate.getInput(), arg); if (null != colOrigs) { for (RelColumnOrigin colOrig : colOrigs) {
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { JoinAlgorithm oldAlgo = join.getJoinAlgorithm(); join.setJoinAlgorithm(TezCommonJoinAlgorithm.INSTANCE); final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); final Double memoryWithinPhase = mq.cumulativeMemoryWithinPhase(join); final Integer splitCount = mq.splitCount(join); join.setJoinAlgorithm(oldAlgo); if (memoryWithinPhase == null || splitCount == null) { return null; } return memoryWithinPhase / splitCount; }
protected FilterSelectivityEstimator(RelNode childRel, RelMetadataQuery mq) { super(true); this.mq = mq; this.childRel = childRel; this.childCardinality = mq.getRowCount(childRel); }
@Override public List<Double> averageColumnSizes(SemiJoin rel, RelMetadataQuery mq) { final RelNode left = rel.getLeft(); final List<Double> lefts = mq.getAverageColumnSizes(left); if (lefts == null) { return null; } final int fieldCount = rel.getRowType().getFieldCount(); Double[] sizes = new Double[fieldCount]; if (lefts != null) { lefts.toArray(sizes); } return ImmutableNullableList.copyOf(sizes); }
@Override public double estimateRowCount(RelMetadataQuery mq) { return mq.getDistinctRowCount(this, groupSet, getCluster().getRexBuilder().makeLiteral(true)); }
public ImmutableBitSet getSortedInputs() throws CalciteSemanticException { ImmutableBitSet.Builder sortedInputsBuilder = ImmutableBitSet.builder(); JoinPredicateInfo joinPredInfo = HiveCalciteUtil.JoinPredicateInfo. constructJoinPredicateInfo(this); List<ImmutableIntList> joinKeysInChildren = new ArrayList<ImmutableIntList>(); joinKeysInChildren.add( ImmutableIntList.copyOf( joinPredInfo.getProjsFromLeftPartOfJoinKeysInChildSchema())); joinKeysInChildren.add( ImmutableIntList.copyOf( joinPredInfo.getProjsFromRightPartOfJoinKeysInChildSchema())); final RelMetadataQuery mq = this.left.getCluster().getMetadataQuery(); for (int i=0; i<this.getInputs().size(); i++) { boolean correctOrderFound = RelCollations.contains( mq.collations(this.getInputs().get(i)), joinKeysInChildren.get(i)); if (correctOrderFound) { sortedInputsBuilder.set(i); } } return sortedInputsBuilder.build(); }
private boolean isSimple(ImmutableBitSet condBits, Filter filter, RelMetadataQuery mq) { // Returns whether the filter is only applied on the key columns if (constraintsBased) { return mq.areColumnsUnique(filter, condBits); } return isKey(condBits, filter, mq); }
@SuppressWarnings("deprecation") public boolean isKey(ImmutableBitSet columns) { final RelMetadataQuery mq = RelMetadataQuery.instance(); return Boolean.TRUE.equals(mq.areColumnsUnique(this, columns)); }
Join join = call.rel(0); RelOptPredicateList preds = RelMetadataQuery.instance().getPulledUpPredicates(join);