private void enableStreamAggregateIfBeneficial(Cuboid cuboid, Set<TblColRef> groupsD, StorageContext context) { CubeDesc cubeDesc = cuboid.getCubeDesc(); boolean enabled = cubeDesc.getConfig().isStreamAggregateEnabled(); Set<TblColRef> shardByInGroups = Sets.newHashSet(); for (TblColRef col : cubeDesc.getShardByColumns()) { if (groupsD.contains(col)) { shardByInGroups.add(col); } } if (!shardByInGroups.isEmpty()) { enabled = false; logger.debug("Aggregate partition results is not beneficial because shard by columns in groupD: {0}", shardByInGroups); } if (!context.isNeedStorageAggregation()) { enabled = false; logger.debug("Aggregate partition results is not beneficial because no storage aggregation"); } if (enabled) { context.enableStreamAggregate(); } }
private boolean isExactAggregation(StorageContext context, Cuboid cuboid, Collection<TblColRef> groups, Set<TblColRef> othersD, Set<TblColRef> singleValuesD, Set<TblColRef> derivedPostAggregation, Collection<FunctionDesc> functionDescs, List<SQLDigest.SQLCall> aggrSQLCalls, boolean groupByExpression) { if (context.isNeedStorageAggregation()) { logger.info("exactAggregation is false because need storage aggregation"); return false;
"Cuboid identified: cube={}, cuboidId={}, groupsD={}, filterD={}, limitPushdown={}, limitLevel={}, storageAggr={}", cubeInstance.getName(), cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(), context.getStorageLimitLevel(), context.isNeedStorageAggregation());
public GTScanRequest planScanRequest() { GTScanRequest scanRequest; List<GTScanRange> scanRanges = this.planScanRanges(); if (scanRanges != null && !scanRanges.isEmpty()) { scanRequest = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions) .setAggrGroupBy(gtAggrGroups).setAggrMetrics(gtAggrMetrics).setAggrMetricsFuncs(gtAggrFuncs) .setFilterPushDown(gtFilter)// .setRtAggrMetrics(gtRtAggrMetrics).setDynamicColumns(gtDynColumns) .setExprsPushDown(tupleExpressionMap)// .setAllowStorageAggregation(context.isNeedStorageAggregation()) .setAggCacheMemThreshold(cubeSegment.getConfig().getQueryCoprocessorMemGB())// .setStoragePushDownLimit(context.getFinalPushDownLimit()) .setStorageLimitLevel(context.getStorageLimitLevel()).setHavingFilterPushDown(havingFilter) .createGTScanRequest(); } else { scanRequest = null; } return scanRequest; }
private void enableStreamAggregateIfBeneficial(Cuboid cuboid, Set<TblColRef> groupsD, StorageContext context) { CubeDesc cubeDesc = cuboid.getCubeDesc(); boolean enabled = cubeDesc.getConfig().isStreamAggregateEnabled(); Set<TblColRef> shardByInGroups = Sets.newHashSet(); for (TblColRef col : cubeDesc.getShardByColumns()) { if (groupsD.contains(col)) { shardByInGroups.add(col); } } if (!shardByInGroups.isEmpty()) { enabled = false; logger.debug("Aggregate partition results is not beneficial because shard by columns in groupD: {0}", shardByInGroups); } if (!context.isNeedStorageAggregation()) { enabled = false; logger.debug("Aggregate partition results is not beneficial because no storage aggregation"); } if (enabled) { context.enableStreamAggregate(); } }
private boolean isExactAggregation(StorageContext context, Cuboid cuboid, Collection<TblColRef> groups, Set<TblColRef> othersD, Set<TblColRef> singleValuesD, Set<TblColRef> derivedPostAggregation, Collection<FunctionDesc> functionDescs, List<SQLDigest.SQLCall> aggrSQLCalls, boolean groupByExpression) { if (context.isNeedStorageAggregation()) { logger.info("exactAggregation is false because need storage aggregation"); return false;
"Cuboid identified: cube={}, cuboidId={}, groupsD={}, filterD={}, limitPushdown={}, limitLevel={}, storageAggr={}", cubeInstance.getName(), cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(), context.getStorageLimitLevel(), context.isNeedStorageAggregation());
public GTScanRequest planScanRequest() { GTScanRequest scanRequest; List<GTScanRange> scanRanges = this.planScanRanges(); if (scanRanges != null && !scanRanges.isEmpty()) { scanRequest = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions) .setAggrGroupBy(gtAggrGroups).setAggrMetrics(gtAggrMetrics).setAggrMetricsFuncs(gtAggrFuncs) .setFilterPushDown(gtFilter)// .setRtAggrMetrics(gtRtAggrMetrics).setDynamicColumns(gtDynColumns) .setExprsPushDown(tupleExpressionMap)// .setAllowStorageAggregation(context.isNeedStorageAggregation()) .setAggCacheMemThreshold(cubeSegment.getConfig().getQueryCoprocessorMemGB())// .setStoragePushDownLimit(context.getFinalPushDownLimit()) .setStorageLimitLevel(context.getStorageLimitLevel()).setHavingFilterPushDown(havingFilter) .createGTScanRequest(); } else { scanRequest = null; } return scanRequest; }