private GroupByOperator opToPush(GroupByOperator gbyOp, BookkeepingInfo bi, IOptimizationContext context) throws AlgebricksException { SourceLocation sourceLoc = gbyOp.getSourceLocation(); Mutable<ILogicalOperator> opRef3 = gbyOp.getInputs().get(0); ILogicalOperator op3 = opRef3.getValue(); GroupByOperator newGbyOp = new GroupByOperator(); newGbyOp.setSourceLocation(sourceLoc); newGbyOp.getInputs().add(new MutableObject<ILogicalOperator>(op3)); Map<String, Object> annotations = newGbyOp.getAnnotations(); annotations.putAll(gbyOp.getAnnotations()); List<LogicalVariable> gbyVars = gbyOp.getGbyVarList(); for (ILogicalPlan nestedPlan : gbyOp.getNestedPlans()) { ILogicalPlan copiedNestedPlan = OperatorManipulationUtil.deepCopy(nestedPlan, gbyOp); OperatorManipulationUtil.computeTypeEnvironment(copiedNestedPlan, context); for (ILogicalPlan p : gbyOp.getNestedPlans()) { gbyOp.getNestedPlans().clear(); gbyOp.getNestedPlans().addAll(copiedNestedPlans); return null; newGbyOp.getNestedPlans().add(pushedSubplan); VariableReferenceExpression varRef = new VariableReferenceExpression(newOpGbyList.get(i)); varRef.setSourceLocation(sourceLoc); newGbyOp.addGbyExpression(replGbyList.get(i), varRef);
@Override public ILogicalOperator visitGroupByOperator(GroupByOperator op, ILogicalOperator arg) throws AlgebricksException { List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByListCopy = deepCopyVariableExpressionReferencePairList(op.getGroupByList()); List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorListCopy = deepCopyVariableExpressionReferencePairList(op.getDecorList()); List<ILogicalPlan> nestedPlansCopy = new ArrayList<ILogicalPlan>(); GroupByOperator opCopy = new GroupByOperator(groupByListCopy, decorListCopy, nestedPlansCopy, op.isGroupAll()); deepCopyInputsAnnotationsAndExecutionMode(op, arg, opCopy); deepCopyPlanList(op.getNestedPlans(), nestedPlansCopy, opCopy); return opCopy; }
@Override public void recomputeSchema() { super.recomputeSchema(); for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) { schema.add(p.first); } for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) { schema.add(getDecorVariable(p)); } }
private GroupByOperator getGroupByOperator(LogicalVariable outputVariable, ILogicalExpression variableRef) { GroupByOperator op = new GroupByOperator(); op.addGbyExpression(outputVariable, variableRef); return op; }
private GroupByOperator makeGroupByOperator(AbstractLogicalOperator.ExecutionMode exeMode) { GroupByOperator groupByOperator = new GroupByOperator(); groupByOperator.setExecutionMode(exeMode); return groupByOperator; }
GroupByOperator gbyOp = new GroupByOperator(); gbyOp.setSourceLocation(sourceLoc); VariableReferenceExpression keyVarRef = new VariableReferenceExpression(keyVar); keyVarRef.setSourceLocation(sourceLoc); gbyOp.getGroupByList().add(new Pair<>(newVar, new MutableObject<>(keyVarRef))); updateInputToOutputVarMapping(keyVar, newVar, false); gbyOp.getInputs().add(new MutableObject<>(inputOp)); gbyOp.getNestedPlans().add(nestedPlan);
ExecutionMode executionMode = gbyOp.getExecutionMode(); if (executionMode != ExecutionMode.PARTITIONED && !(executionMode == ExecutionMode.UNPARTITIONED && gbyOp.isGroupAll())) { return false; for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyOp.getDecorList()) { List<LogicalVariable> usedDecorVars = new ArrayList<LogicalVariable>(); newGbyOp.addDecorExpression(null, p.second.getValue()); newGbyLiveVars.add(usedVar); newGbyOp.setExecutionMode(ExecutionMode.LOCAL); Object v = gbyOp.getAnnotations().get(OperatorAnnotations.USE_HASH_GROUP_BY); newGbyOp.getAnnotations().put(OperatorAnnotations.USE_HASH_GROUP_BY, v); Object v2 = gbyOp.getAnnotations().get(OperatorAnnotations.USE_EXTERNAL_GROUP_BY); newGbyOp.getAnnotations().put(OperatorAnnotations.USE_EXTERNAL_GROUP_BY, v2); LogicalVariable newDecorVar = context.newVar(); VariableReferenceExpression varRef = new VariableReferenceExpression(var); varRef.setSourceLocation(gbyOp.getSourceLocation()); newGbyOp.addDecorExpression(newDecorVar, varRef); VariableUtilities.substituteVariables(gbyOp.getNestedPlans().get(0).getRoots().get(0).getValue(), var, newDecorVar, context); Mutable<ILogicalOperator> opRef3 = gbyOp.getInputs().get(0); opRef3.setValue(newGbyOp); typeGby(newGbyOp, context);
GroupByOperator gOp = new GroupByOperator(); for (GbyVariableExpressionPair ve : gc.getGbyPairList()) { VariableExpr vexpr = ve.getVar(); LogicalVariable v = vexpr == null ? context.newVar() : context.newVarFromExpression(vexpr); Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = langExprToAlgExpression(ve.getExpr(), topOp); gOp.addGbyExpression(v, eo.first); topOp = eo.second; LogicalVariable v = vexpr == null ? context.newVar() : context.newVarFromExpression(vexpr); Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = langExprToAlgExpression(ve.getExpr(), topOp); gOp.addDecorExpression(v, eo.first); topOp = eo.second; gOp.getInputs().add(topOp); ILogicalPlan nestedPlan = createNestedPlanWithAggregate(groupLogicalVar, BuiltinFunctions.LISTIFY, groupRecordVarRef, new MutableObject<>(ntsOp)); gOp.getNestedPlans().add(nestedPlan); context.setVar(groupVar, groupLogicalVar); ILogicalPlan nestedPlan = createNestedPlanWithAggregate(withLogicalVar, BuiltinFunctions.LISTIFY, listifyInput.first, listifyInput.second); gOp.getNestedPlans().add(nestedPlan); context.setVar(withVar, withLogicalVar); gOp.setGroupAll(gc.isGroupAll()); gOp.getAnnotations().put(OperatorAnnotations.USE_HASH_GROUP_BY, gc.hasHashGroupByHint()); gOp.setSourceLocation(sourceLoc); return new Pair<>(gOp, null);
int keys[] = JobGenHelper.variablesToFieldIndexes(gbyCols, inputSchemas[0]); GroupByOperator gby = (GroupByOperator) op; int numFds = gby.getDecorList().size(); int fdColumns[] = new int[numFds]; int j = 0; for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) { ILogicalExpression expr = p.second.getValue(); if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) { if (gby.getNestedPlans().size() != 1) { throw new AlgebricksException( "External group-by currently works only for one nested plan with one root containing" + "an aggregate and a nested-tuple-source."); ILogicalPlan p0 = gby.getNestedPlans().get(0); if (p0.getRoots().size() != 1) { throw new AlgebricksException( for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getGroupByList()) { keyAndDecVariables.add(p.first); for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) { keyAndDecVariables.add(GroupByOperator.getDecorVariable(p)); aggregatorFactory.setSourceLocation(gby.getSourceLocation()); AbstractAggregatorDescriptorFactory mergeFactory = new SerializableAggregatorDescriptorFactory(merges); mergeFactory.setSourceLocation(gby.getSourceLocation()); int groupByColumnsCount = gby.getGroupByList().size() + numFds;
if (op.getOperatorTag().equals(LogicalOperatorTag.GROUP)) { GroupByOperator gby = (GroupByOperator) op; if (gby.getNestedPlans().size() == 1) { ILogicalPlan p0 = gby.getNestedPlans().get(0); if (p0.getRoots().size() == 1) { Mutable<ILogicalOperator> r0 = p0.getRoots().get(0); if ((gby.getAnnotations().get(OperatorAnnotations.USE_HASH_GROUP_BY) == Boolean.TRUE || gby .getAnnotations().get(OperatorAnnotations.USE_EXTERNAL_GROUP_BY) == Boolean.TRUE)) { boolean setToExternalGby = false; if (serializable) { gby.getGroupByList(), physicalOptimizationConfig.getMaxFramesForGroupBy(), (long) physicalOptimizationConfig.getMaxFramesForGroupBy() List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList = gby.getGroupByList(); List<LogicalVariable> columnList = new ArrayList<LogicalVariable>(gbyList.size()); for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) { op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList, gby.isGroupAll(), context.getPhysicalOptimizationConfig().getMaxFramesForGroupBy())); List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList = gby.getGroupByList(); List<LogicalVariable> columnList = new ArrayList<LogicalVariable>(gbyList.size()); for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) { op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList, gby.isGroupAll(), context.getPhysicalOptimizationConfig().getMaxFramesForGroupBy())); } else {
for (ILogicalPlan p : op.getNestedPlans()) { for (Mutable<ILogicalOperator> r : p.getRoots()) { ILogicalOperator op2 = r.getValue(); ILogicalOperator op0 = op.getInputs().get(0).getValue(); inheritedFDs.addAll(getOrComputeFDs(op0, ctx)); Map<LogicalVariable, EquivalenceClass> inheritedEcs = getOrComputeEqClasses(op0, ctx); List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gByList = op.getGroupByList(); for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) { premiseGby.add(p.first); List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList = op.getDecorList(); conclDecor.add(GroupByOperator.getDecorVariable(p)); LogRedactionUtil.userData(GroupByOperator.veListToString(gByList)), LogRedactionUtil.userData(GroupByOperator.veListToString(newGbyList)));
exchange.getInputs().add(new MutableObject<>(scanOperator)); GroupByOperator groupByOperator = new GroupByOperator(); groupByOperator.setExecutionMode(PARTITIONED); groupByOperator .setPhysicalOperator(new PreclusteredGroupByPOperator(Collections.emptyList(), true, FRAME_LIMIT)); groupByOperator.getInputs().add(new MutableObject<>(exchange));
@Override public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException { for (ILogicalPlan p : op.getNestedPlans()) { for (Mutable<ILogicalOperator> r : p.getRoots()) { VariableUtilities.getLiveVariables(r.getValue(), producedVariables); } } for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) { if (p.first != null) { producedVariables.add(p.first); } } for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) { if (p.first != null) { producedVariables.add(p.first); } } return null; }
if (!groupOp.isGroupAll()) { return false; List<LogicalVariable> groupVars = groupOp.getGbyVarList(); List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList = groupOp.getDecorList(); if (!groupVars.isEmpty() || !decorList.isEmpty()) { return false; List<ILogicalPlan> nestedPlans = groupOp.getNestedPlans(); if (nestedPlans.size() > 1) { return false; nestedTupleSourceRef.setValue(groupOp.getInputs().get(0).getValue()); return true;
GroupByOperator groupbyOp = new GroupByOperator(groupByList, groupByDecorList, subplanOp.getNestedPlans()); groupbyOp.setSourceLocation(sourceLoc); groupByDecorList.add(new Pair<>(null, new MutableObject<>(liveVarRef))); groupbyOp.getInputs().add(new MutableObject<>(topJoinRef.getValue()));
private void mapVariablesForGroupBy(ILogicalOperator left, ILogicalOperator right) { if (left.getOperatorTag() != right.getOperatorTag()) { return; } GroupByOperator leftOp = (GroupByOperator) left; GroupByOperator rightOp = (GroupByOperator) right; List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> leftPairs = leftOp.getGroupByList(); List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> rightPairs = rightOp.getGroupByList(); mapVarExprPairList(leftPairs, rightPairs); leftPairs = leftOp.getDecorList(); rightPairs = rightOp.getDecorList(); mapVarExprPairList(leftPairs, rightPairs); }
if (gby.getNestedPlans().size() != 1) { return false; if (gby.getNestedPlans().get(0).getRoots().size() != 1) { return false; (AbstractLogicalOperator) gby.getNestedPlans().get(0).getRoots().get(0).getValue(); if (nestedPlanRoot.getOperatorTag() != LogicalOperatorTag.AGGREGATE) { return false; for (int i = 0; i < gby.getGroupByList().size(); i++) { if (gby.getGroupByList().get(i).first != null) { gbyKeyAssgnVars.add(gby.getGroupByList().get(i).first); gbyKeyAssgnExprs.add(gby.getGroupByList().get(i).second); assign.getInputs().add(aggInputOpRef); AssignOperator gbyKeyAssign = new AssignOperator(gbyKeyAssgnVars, gbyKeyAssgnExprs); gbyKeyAssign.setSourceLocation(gby.getSourceLocation()); gbyKeyAssign.getInputs().add(gby.getInputs().get(0)); bottomOpRef.setValue(gbyKeyAssign); raggOp.setPhysicalOperator(raggPOp); raggOp.getInputs().add(nestedPlanRoot.getInputs().get(0)); gby.getNestedPlans().get(0).getRoots().set(0, new MutableObject<ILogicalOperator>(raggOp));
@Override public String visitGroupByOperator(GroupByOperator op, Boolean showDetails) throws AlgebricksException { stringBuilder.setLength(0); stringBuilder.append("group by").append(op.isGroupAll() ? " (all)" : "").append(" ("); printVariableAndExprList(op.getGroupByList()); stringBuilder.append(") decor ("); printVariableAndExprList(op.getDecorList()); stringBuilder.append(")"); appendSchema(op, showDetails); appendAnnotations(op, showDetails); appendPhysicalOperatorInfo(op, showDetails); return stringBuilder.toString(); }
GroupByOperator gby = (GroupByOperator) op; StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1]; if (gby.isGroupAll() && gby.isGlobal()) { if (op.getExecutionMode() == ExecutionMode.UNPARTITIONED) { pv[0] = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, null); for (ILogicalPlan p : gby.getNestedPlans()) { for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorPair : gby.getDecorList()) { List<LogicalVariable> hd = gby.getGbyVarList(); List<LogicalVariable> tl = new ArrayList<>(); tl.add(((VariableReferenceExpression) decorPair.second.getValue()).getVariableReference());
private Map<LogicalVariable, LogicalVariable> buildVarExprList(Collection<LogicalVariable> vars, IOptimizationContext context, GroupByOperator g, List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> outVeList) throws AlgebricksException { SourceLocation sourceLoc = g.getSourceLocation(); Map<LogicalVariable, LogicalVariable> m = new HashMap<LogicalVariable, LogicalVariable>(); for (LogicalVariable ov : vars) { LogicalVariable newVar = context.newVar(); ILogicalExpression varExpr = new VariableReferenceExpression(newVar); ((VariableReferenceExpression) varExpr).setSourceLocation(sourceLoc); outVeList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(ov, new MutableObject<ILogicalExpression>(varExpr))); for (ILogicalPlan p : g.getNestedPlans()) { for (Mutable<ILogicalOperator> r : p.getRoots()) { OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), ov, newVar, true, context); } } AbstractLogicalOperator opUnder = (AbstractLogicalOperator) g.getInputs().get(0).getValue(); OperatorManipulationUtil.substituteVarRec(opUnder, ov, newVar, true, context); m.put(ov, newVar); } return m; } }