private boolean isInvertedIndexSearch(AbstractUnnestMapOperator op) { IPhysicalOperator physicalOperator = op.getPhysicalOperator(); final PhysicalOperatorTag physicalOperatorTag = physicalOperator.getOperatorTag(); if (physicalOperatorTag == PhysicalOperatorTag.LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH || physicalOperatorTag == PhysicalOperatorTag.SINGLE_PARTITION_INVERTED_INDEX_SEARCH) { return true; } return false; }
private boolean isBlockingGroupBy(GroupByOperator op) { return op.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.EXTERNAL_GROUP_BY || op.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.SORT_GROUP_BY; }
private long getExchangeRequiredMemory(ExchangeOperator op) { final IPhysicalOperator physicalOperator = op.getPhysicalOperator(); final PhysicalOperatorTag physicalOperatorTag = physicalOperator.getOperatorTag(); if (physicalOperatorTag == PhysicalOperatorTag.ONE_TO_ONE_EXCHANGE || physicalOperatorTag == PhysicalOperatorTag.SORT_MERGE_EXCHANGE) { return getOperatorRequiredMemory(op, frameSize); } return 2L * MAX_BUFFER_PER_CONNECTION * numComputationPartitions * numComputationPartitions * frameSize; }
private void calculateMemoryUsageForExchange(ExchangeOperator op) throws AlgebricksException { visitInternal(op, false); IPhysicalOperator physicalOperator = op.getPhysicalOperator(); PhysicalOperatorTag physicalOperatorTag = physicalOperator.getOperatorTag(); if (physicalOperatorTag == PhysicalOperatorTag.ONE_TO_ONE_EXCHANGE || physicalOperatorTag == PhysicalOperatorTag.SORT_MERGE_EXCHANGE) { addOutputBuffer(op); return; } stageMemorySoFar += 2L * MAX_BUFFER_PER_CONNECTION * numComputationPartitions * numComputationPartitions * frameSize; clusterCapacity.setAggregatedMemoryByteSize(stageMemorySoFar); }
private void validateOperatorStages(List<PlanStage> stages, ILogicalOperator operator) { if (stages.size() == 1) { return; } long expectedAppearances = BLOCKING_OPERATORS.contains(operator.getOperatorTag()) ? 2 : 1; if (operator.getOperatorTag() == GROUP) { GroupByOperator groupByOperator = (GroupByOperator) operator; if (groupByOperator.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.EXTERNAL_GROUP_BY || groupByOperator.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.SORT_GROUP_BY) { expectedAppearances = 2; } } final long actual = stages.stream().map(PlanStage::getOperators).filter(op -> op.contains(operator)).count(); Assert.assertEquals(expectedAppearances, actual); }
if (!(op.getPhysicalOperator().getOperatorTag() == aop.getPhysicalOperator().getOperatorTag())) { return Boolean.FALSE;
/** * When true, the sort operator requires ORDERED_PARTITION (only applicable to dynamic version for now). * Conditions: * 1. Execution mode == partitioned * 2. Dynamic range map was not disabled by some checks * 3. User didn't disable it * 4. User didn't provide static range map * 5. Physical sort operator is not in-memory * 6. There are at least two partitions in the cluster * @param sortOp the sort operator * @param clusterDomain the partitions specification of the cluster * @param ctx optimization context * @return true if the sort operator should be full parallel sort, false otherwise. */ private boolean isFullParallel(AbstractLogicalOperator sortOp, INodeDomain clusterDomain, IOptimizationContext ctx) { return sortOp.getAnnotations().get(OperatorAnnotations.USE_DYNAMIC_RANGE) != Boolean.FALSE && !sortOp.getAnnotations().containsKey(OperatorAnnotations.USE_STATIC_RANGE) && sortOp.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.STABLE_SORT && clusterDomain.cardinality() != null && clusterDomain.cardinality() > 1 && ctx.getPhysicalOptimizationConfig().getSortParallel(); } }
private boolean isRedundantSort(Mutable<ILogicalOperator> opRef, IPhysicalPropertiesVector delivered, IPhysicalPropertiesVector diffOfProperties, IOptimizationContext context) { AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); if (op.getOperatorTag() != LogicalOperatorTag.ORDER || (op.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.STABLE_SORT && op.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.IN_MEMORY_STABLE_SORT) || delivered.getLocalProperties() == null) { return false; } AbstractStableSortPOperator sortOp = (AbstractStableSortPOperator) op.getPhysicalOperator(); sortOp.computeLocalProperties(op); ILocalStructuralProperty orderProp = sortOp.getOrderProperty(); return PropertiesUtil.matchLocalProperties(Collections.singletonList(orderProp), delivered.getLocalProperties(), context.getEquivalenceClassMap(op), context.getFDList(op)); }
|| aop.getPhysicalOperator().getOperatorTag() != op.getPhysicalOperator().getOperatorTag()) { return Boolean.FALSE;
switch (physOp.getOperatorTag()) { case BROADCAST_EXCHANGE: case ONE_TO_ONE_EXCHANGE: break; default: throw new AlgebricksException("Unhandled physical operator tag '" + physOp.getOperatorTag() + "'.");
AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue(); if (op1.getPhysicalOperator() == null || (op1.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.HASH_PARTITION_EXCHANGE && op1 .getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.HASH_PARTITION_MERGE_EXCHANGE)) { return false; if (physicalOp == null) { return false; } else if (physicalOp.getOperatorTag() == PhysicalOperatorTag.SORT_MERGE_EXCHANGE) { break; } else if (!currentOp.isMap() || currentOp.getOperatorTag() == LogicalOperatorTag.UNNEST if (op1.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.HASH_PARTITION_MERGE_EXCHANGE) {
private void optimizeUsingConstraintsAndEquivClasses(AbstractLogicalOperator op) { IPhysicalOperator pOp = op.getPhysicalOperator(); switch (pOp.getOperatorTag()) { case HASH_GROUP_BY: case EXTERNAL_GROUP_BY: { GroupByOperator gby = (GroupByOperator) op; ExternalGroupByPOperator hgbyOp = (ExternalGroupByPOperator) pOp; hgbyOp.computeColumnSet(gby.getGroupByList()); break; } case PRE_CLUSTERED_GROUP_BY: case MICRO_PRE_CLUSTERED_GROUP_BY: { GroupByOperator gby = (GroupByOperator) op; AbstractPreclusteredGroupByPOperator preSortedGby = (AbstractPreclusteredGroupByPOperator) pOp; preSortedGby.setGbyColumns(gby.getGbyVarList()); break; } case PRE_SORTED_DISTINCT_BY: case MICRO_PRE_SORTED_DISTINCT_BY: { DistinctOperator d = (DistinctOperator) op; AbstractPreSortedDistinctByPOperator preSortedDistinct = (AbstractPreSortedDistinctByPOperator) pOp; preSortedDistinct.setDistinctByColumns(d.getDistinctByVarList()); break; } default: } }
AbstractLogicalOperator op = (AbstractLogicalOperator) childRef.getValue(); if (op.getOperatorTag() == LogicalOperatorTag.GROUP) { PhysicalOperatorTag opTag = op.getPhysicalOperator().getOperatorTag(); GroupByOperator groupByOperator = (GroupByOperator) op; if (opTag == PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY) { Mutable<ILogicalOperator> op2Ref = op.getInputs().get(0).getValue().getInputs().get(0); AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue(); if (op2.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.STABLE_SORT) { AbstractStableSortPOperator sortPhysicalOperator = (AbstractStableSortPOperator) op2.getPhysicalOperator();
ILogicalOperator childOp = parentOp.getOperatorTag() == LogicalOperatorTag.PROJECT ? assignOperator : projectOperator; if (!HeuristicOptimizer.isHyracksOp(parentOp.getPhysicalOperator().getOperatorTag())) { parentOp.getInputs().set(index, new MutableObject<ILogicalOperator>(childOp)); } else {
private boolean testIfExchangeBelow(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); boolean exchInserted = false; for (Mutable<ILogicalOperator> i : op.getInputs()) { AbstractLogicalOperator c = (AbstractLogicalOperator) i.getValue(); if (c.getOperatorTag() != LogicalOperatorTag.EXCHANGE) { if (c.getPhysicalOperator() == null) { return false; } insertOneToOneExchange(i, context); exchInserted = true; } } IPhysicalOperator pt = op.getPhysicalOperator(); if (pt.isJobGenDisabledBelowMe() || arrayContains(operatorsBelowWhichJobGenIsDisabled, pt.getOperatorTag())) { for (Mutable<ILogicalOperator> i : op.getInputs()) { disableJobGenRec(i.getValue()); } } return exchInserted; }
/** * Generate new ORDER operator that uses TopKSort module and replaces the old ORDER operator. */ private boolean pushLimitIntoOrder(Mutable<ILogicalOperator> opRef, Mutable<ILogicalOperator> opRef2, IOptimizationContext context) throws AlgebricksException { PhysicalOptimizationConfig physicalOptimizationConfig = context.getPhysicalOptimizationConfig(); LimitOperator limitOp = (LimitOperator) opRef.getValue(); OrderOperator orderOp = (OrderOperator) opRef2.getValue(); // We don't push-down LIMIT into in-memory sort. if (orderOp.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.STABLE_SORT) { return false; } Integer topK = getOutputLimit(limitOp); if (topK == null) { return false; } // Create the new ORDER operator, set the topK value, and replace the current one. OrderOperator newOrderOp = new OrderOperator(orderOp.getOrderExpressions(), topK); newOrderOp.setSourceLocation(orderOp.getSourceLocation()); newOrderOp.setPhysicalOperator( new StableSortPOperator(physicalOptimizationConfig.getMaxFramesExternalSort(), newOrderOp.getTopK())); newOrderOp.getInputs().addAll(orderOp.getInputs()); newOrderOp.setExecutionMode(orderOp.getExecutionMode()); newOrderOp.recomputeSchema(); newOrderOp.computeDeliveredPhysicalProperties(context); opRef2.setValue(newOrderOp); context.computeAndSetTypeEnvironmentForOperator(newOrderOp); context.addToDontApplySet(this, limitOp); return true; }