private DrillDistributionTrait convertDist(DrillDistributionTrait srcDist, Map<Integer, Integer> inToOut) { List<DistributionField> newFields = Lists.newArrayList(); for (DistributionField field : srcDist.getFields()) { if (inToOut.containsKey(field.getFieldId())) { newFields.add(new DistributionField(inToOut.get(field.getFieldId()))); } } // After the projection, if the new distribution fields is empty, or new distribution fields is a subset of // original distribution field, we should replace with either SINGLETON or RANDOM_DISTRIBUTED. if (newFields.isEmpty() || newFields.size() < srcDist.getFields().size()) { if (srcDist.getType() != DistributionType.SINGLETON) { return DrillDistributionTrait.RANDOM_DISTRIBUTED; } else { return DrillDistributionTrait.SINGLETON; } } else { return new DrillDistributionTrait(srcDist.getType(), ImmutableList.copyOf(newFields)); } }
@Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj instanceof DrillDistributionTrait) { DrillDistributionTrait that = (DrillDistributionTrait) obj; return this.type == that.type && this.fields.equals(that.fields) && arePartitionFunctionsSame(this.partitionFunction, that.partitionFunction); } return false; }
@Override public boolean satisfies(RelTrait trait) { if (trait instanceof DrillDistributionTrait) { DistributionType requiredDist = ((DrillDistributionTrait) trait).getType(); if (requiredDist == DistributionType.ANY) { return true; } if (this.type == DistributionType.HASH_DISTRIBUTED) { if (requiredDist == DistributionType.HASH_DISTRIBUTED) { // A subset of the required distribution columns can satisfy (subsume) the requirement // e.g: required distribution: {a, b, c} // Following can satisfy the requirements: {a}, {b}, {c}, {a, b}, {b, c}, {a, c} or {a, b, c} // New: Use equals for subsumes check of hash distribution. If we uses subsumes, // a join may end up with hash-distributions using different keys. This would // cause incorrect query result. return this.equals(trait); } else if (requiredDist == DistributionType.RANDOM_DISTRIBUTED) { return true; // hash distribution subsumes random distribution and ANY distribution } } if(this.type == DistributionType.RANGE_DISTRIBUTED) { if (requiredDist == DistributionType.RANDOM_DISTRIBUTED) { return true; // RANGE_DISTRIBUTED distribution subsumes random distribution and ANY distribution } } } return this.equals(trait); }
if (currentDist.equals(toDist)) { return rel; if (currentDist.equals(DrillDistributionTrait.DEFAULT) && !(rel instanceof RelSubset) ) { return null; switch(toDist.getType()){ case HASH_DISTRIBUTED: return new HashToRandomExchangePrel(rel.getCluster(), planner.emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(toDist), rel, toDist.getFields()); case RANGE_DISTRIBUTED: toDist.getFields(), toDist.getPartitionFunction()); case BROADCAST_DISTRIBUTED: return new BroadcastExchangePrel(rel.getCluster(), planner.emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(toDist), rel);
private DrillDistributionTrait getDistribution(List<Integer> keys) { List<DistributionField> fields = Lists.newArrayList(); for (int key : keys) { fields.add(new DistributionField(key)); } return new DrillDistributionTrait(DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(fields)); }
private boolean isDefaultDist(RelNode n) { return n.getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE).equals(DrillDistributionTrait.DEFAULT); }
public static RelNode getExchange(RelOptCluster cluster, boolean isSingleton, boolean isExchangeRequired, RelTraitSet traits, DrillDistributionTrait distributionTrait, IndexCallContext indexContext, RelNode input) { if (!isExchangeRequired) { return input; } if (isSingleton) { return new SingleMergeExchangePrel(cluster, traits.replace(DrillDistributionTrait.SINGLETON), input, indexContext.getCollation()); } else { return new HashToMergeExchangePrel(cluster, traits.replace(distributionTrait), input, distributionTrait.getFields(), indexContext.getCollation(), PrelUtil.getSettings(cluster).numEndPoints()); } }
protected enum PhysicalJoinType {HASH_JOIN, MERGE_JOIN, NESTEDLOOP_JOIN}
protected boolean checkBroadcastConditions(RelOptPlanner planner, DrillJoin join, RelNode left, RelNode right) { double estimatedRightRowCount = RelMetadataQuery.instance().getRowCount(right); if (estimatedRightRowCount < PrelUtil.getSettings(join.getCluster()).getBroadcastThreshold() && ! left.getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE).equals(DrillDistributionTrait.SINGLETON) && (join.getJoinType() == JoinRelType.INNER || join.getJoinType() == JoinRelType.LEFT) ) { return true; } return false; }
private static boolean validateTraits(RelTraitSet traitSet, RelNode left, RelNode right) { ImmutableBitSet bitSet = ImmutableBitSet.range(left.getRowType().getFieldCount(), left.getRowType().getFieldCount() + right.getRowType().getFieldCount()); for (RelTrait trait: traitSet) { if (trait.getTraitDef().getTraitClass().equals(RelCollation.class)) { RelCollation collationTrait = (RelCollation)trait; for (RelFieldCollation field : collationTrait.getFieldCollations()) { if (bitSet.indexOf(field.getFieldIndex()) > 0) { return false; } } } else if (trait.getTraitDef().getTraitClass().equals(DrillDistributionTrait.class)) { DrillDistributionTrait distributionTrait = (DrillDistributionTrait) trait; for (DrillDistributionTrait.DistributionField field : distributionTrait.getFields()) { if (bitSet.indexOf(field.getFieldId()) > 0) { return false; } } } } return true; }
protected RelNode createRangeDistRight(final RelNode rightPrel, final RelDataTypeField rightRowKeyField, final DbGroupScan origDbGroupScan) { List<DrillDistributionTrait.DistributionField> rangeDistFields = Lists.newArrayList(new DrillDistributionTrait.DistributionField(0 /* rowkey ordinal on the right side */)); FieldReference rangeDistRef = FieldReference.getWithQuotedRef(rightRowKeyField.getName()); List<FieldReference> rangeDistRefList = Lists.newArrayList(); rangeDistRefList.add(rangeDistRef); final DrillDistributionTrait distRight; if (IndexPlanUtils.scanIsPartition(origDbGroupScan)) { distRight = new DrillDistributionTrait( DrillDistributionTrait.DistributionType.RANGE_DISTRIBUTED, ImmutableList.copyOf(rangeDistFields), origDbGroupScan.getRangePartitionFunction(rangeDistRefList)); } else { distRight = DrillDistributionTrait.SINGLETON; } RelTraitSet rightTraits = newTraitSet(distRight).plus(Prel.DRILL_PHYSICAL); RelNode convertedRight = Prule.convert(rightPrel, rightTraits); return convertedRight; }
DrillDistributionTrait hashChild = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(childDistFields)); RelTraitSet traitsChild = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(hashChild); convertedChild = convert(child, PrelUtil.fixTraits(call, traitsChild));
settings.getSliceTarget() < indexProjectPrel.getRows()) { final DrillDistributionTrait distRight = new DrillDistributionTrait(DistributionType.BROADCAST_DISTRIBUTED); rightSideTraits = newTraitSet(distRight).plus(Prel.DRILL_PHYSICAL); right.getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE) != DrillDistributionTrait.SINGLETON) { final DrillDistributionTrait distRight = new DrillDistributionTrait(DistributionType.BROADCAST_DISTRIBUTED); rightSideTraits = newTraitSet(distRight).plus(Prel.DRILL_PHYSICAL);
&& windowBase.orderKeys.getFieldCollations().isEmpty()) { DrillDistributionTrait distEmptyKeys = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.SINGLETON); new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionFields(windowBase))); new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionFieldsFromCollation(windowBase)));
rangeDistRefList.add(rangeDistRef); RelNode leftScan = DrillPushRowKeyJoinToScanRule.getValidJoinInput(left); DrillDistributionTrait rangePartRight = new DrillDistributionTrait( DrillDistributionTrait.DistributionType.RANGE_DISTRIBUTED, ImmutableList.copyOf(rangeDistFields),
new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(aggregate, true /* get all grouping keys */))); new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(aggregate, false /* get single grouping key */)));
OrderedRel rel = indexContext.getSort(); DrillDistributionTrait hashDistribution = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(indexContext.getDistributionFields()));
@Override public void onMatch(RelOptRuleCall call) { final DrillSortRel sort = call.rel(0); final RelNode input = sort.getInput(); // Keep the collation in logical sort. Convert input into a RelNode with 1) this collation, 2) Physical, 3) hash distributed on DrillDistributionTrait hashDistribution = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(sort))); final RelTraitSet traits = RelTraitSet.createEmpty().plus(Prel.DRILL_PHYSICAL).plus(hashDistribution); SortPrel child = new SortPrel(sort.getCluster(), traits.plus(sort.getCollation()), convert(sort.getInput(), traits), sort.getCollation(), false); if(isSingleMode(call)){ call.transformTo(child); }else{ RelNode exch = new SingleMergeExchangePrel(sort.getCluster(), sort.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON), child, sort.getCollation()); call.transformTo(exch); // transform logical "sort" into "SingleMergeExchange". } }
new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(aggregate, true))); new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(aggregate, false)));
final RelCollation collationLeft, final RelCollation collationRight) throws InvalidRelException { DrillDistributionTrait distBroadcastRight = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.BROADCAST_DISTRIBUTED); RelTraitSet traitsRight = null; RelTraitSet traitsLeft = left.getTraitSet().plus(Prel.DRILL_PHYSICAL);