private DrillDistributionTrait convertDist(DrillDistributionTrait srcDist, Map<Integer, Integer> inToOut) { List<DistributionField> newFields = Lists.newArrayList(); for (DistributionField field : srcDist.getFields()) { if (inToOut.containsKey(field.getFieldId())) { newFields.add(new DistributionField(inToOut.get(field.getFieldId()))); } } // After the projection, if the new distribution fields is empty, or new distribution fields is a subset of // original distribution field, we should replace with either SINGLETON or RANDOM_DISTRIBUTED. if (newFields.isEmpty() || newFields.size() < srcDist.getFields().size()) { if (srcDist.getType() != DistributionType.SINGLETON) { return DrillDistributionTrait.RANDOM_DISTRIBUTED; } else { return DrillDistributionTrait.SINGLETON; } } else { return new DrillDistributionTrait(srcDist.getType(), ImmutableList.copyOf(newFields)); } }
final int fieldId = distField.getFieldId(); distFieldRefs.add(rexBuilder.makeInputRef(childRowTypeFields.get(fieldId).getType(), fieldId));
childDistFields.add(new DistributionField(f.getIndex()));
int joinKeyRight = join.getRightKeys().get(0); List<DrillDistributionTrait.DistributionField> rangeDistFields = Lists.newArrayList(new DrillDistributionTrait.DistributionField(joinKeyRight /* `rowkey equivalent` ordinal on the right side */)); List<FieldReference> rangeDistRefList = Lists.newArrayList(); FieldReference rangeDistRef =
private static boolean validateTraits(RelTraitSet traitSet, RelNode left, RelNode right) { ImmutableBitSet bitSet = ImmutableBitSet.range(left.getRowType().getFieldCount(), left.getRowType().getFieldCount() + right.getRowType().getFieldCount()); for (RelTrait trait: traitSet) { if (trait.getTraitDef().getTraitClass().equals(RelCollation.class)) { RelCollation collationTrait = (RelCollation)trait; for (RelFieldCollation field : collationTrait.getFieldCollations()) { if (bitSet.indexOf(field.getFieldIndex()) > 0) { return false; } } } else if (trait.getTraitDef().getTraitClass().equals(DrillDistributionTrait.class)) { DrillDistributionTrait distributionTrait = (DrillDistributionTrait) trait; for (DrillDistributionTrait.DistributionField field : distributionTrait.getFields()) { if (bitSet.indexOf(field.getFieldId()) > 0) { return false; } } } } return true; }
/** * Create a distribution hash expression. * * @param fields Distribution fields * @param rowType Row type * @return */ public static LogicalExpression getHashExpression(List<DistributionField> fields, RelDataType rowType) { assert fields.size() > 0; final List<String> childFields = rowType.getFieldNames(); // If we already included a field with hash - no need to calculate hash further down if ( childFields.contains(HASH_EXPR_NAME)) { return new FieldReference(HASH_EXPR_NAME); } final List<LogicalExpression> expressions = new ArrayList<LogicalExpression>(childFields.size()); for(int i =0; i < fields.size(); i++){ expressions.add(new FieldReference(childFields.get(fields.get(i).getFieldId()), ExpressionPosition.UNKNOWN)); } final LogicalExpression distSeed = ValueExpressions.getInt(DIST_SEED); return createHashBasedPartitionExpression(expressions, distSeed, HASH_HELPER_LOGICALEXPRESSION); } }
protected RelNode createRangeDistRight(final RelNode rightPrel, final RelDataTypeField rightRowKeyField, final DbGroupScan origDbGroupScan) { List<DrillDistributionTrait.DistributionField> rangeDistFields = Lists.newArrayList(new DrillDistributionTrait.DistributionField(0 /* rowkey ordinal on the right side */)); FieldReference rangeDistRef = FieldReference.getWithQuotedRef(rightRowKeyField.getName()); List<FieldReference> rangeDistRefList = Lists.newArrayList(); rangeDistRefList.add(rangeDistRef); final DrillDistributionTrait distRight; if (IndexPlanUtils.scanIsPartition(origDbGroupScan)) { distRight = new DrillDistributionTrait( DrillDistributionTrait.DistributionType.RANGE_DISTRIBUTED, ImmutableList.copyOf(rangeDistFields), origDbGroupScan.getRangePartitionFunction(rangeDistRefList)); } else { distRight = DrillDistributionTrait.SINGLETON; } RelTraitSet rightTraits = newTraitSet(distRight).plus(Prel.DRILL_PHYSICAL); RelNode convertedRight = Prule.convert(rightPrel, rightTraits); return convertedRight; }
protected List<DistributionField> getDistributionField(DrillAggregateRel rel, boolean allFields) { List<DistributionField> groupByFields = Lists.newArrayList(); for (int group : remapGroupSet(rel.getGroupSet())) { DistributionField field = new DistributionField(group); groupByFields.add(field); if (!allFields && groupByFields.size() == 1) { // if we are only interested in 1 grouping field, pick the first one for now.. // but once we have num distinct values (NDV) statistics, we should pick the one // with highest NDV. break; } } return groupByFields; }
private List<DistributionField> getDistributionFieldsFromCollation(Window.Group window) { List<DistributionField> distFields = Lists.newArrayList(); for (RelFieldCollation relField : window.collation().getFieldCollations()) { DistributionField field = new DistributionField(relField.getFieldIndex()); distFields.add(field); } return distFields; }
public static List<DrillDistributionTrait.DistributionField> getDistributionField(Sort rel) { List<DrillDistributionTrait.DistributionField> distFields = Lists.newArrayList(); for (RelFieldCollation relField : getCollation(rel).getFieldCollations()) { DrillDistributionTrait.DistributionField field = new DrillDistributionTrait.DistributionField(relField.getFieldIndex()); distFields.add(field); } return distFields; }
private List<DistributionField> getDistributionField(DrillSortRel rel) { List<DistributionField> distFields = Lists.newArrayList(); for (RelFieldCollation relField : rel.getCollation().getFieldCollations()) { DistributionField field = new DistributionField(relField.getFieldIndex()); distFields.add(field); } return distFields; }
private DrillDistributionTrait getDistribution(List<Integer> keys) { List<DistributionField> fields = Lists.newArrayList(); for (int key : keys) { fields.add(new DistributionField(key)); } return new DrillDistributionTrait(DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(fields)); }
private List<DrillDistributionTrait.DistributionField> getDistributionFields(Window.Group window) { List<DrillDistributionTrait.DistributionField> groupByFields = Lists.newArrayList(); for (int group : BitSets.toIter(window.keys)) { DrillDistributionTrait.DistributionField field = new DrillDistributionTrait.DistributionField(group); groupByFields.add(field); } return groupByFields; }
protected List<DistributionField> getDistributionField(List<Integer> keys) { List<DistributionField> distFields = Lists.newArrayList(); for (int key : keys) { distFields.add(new DistributionField(key)); } return distFields; }