/** * This function checks the query for dimensions which can be optimized by applying the dimension extraction * as the final step of the query instead of on every event. * * @param query The query to check for optimizations * * @return A collection of DimensionsSpec which can be extracted at the last second upon query completion. */ public static Collection<DimensionSpec> extractionsToRewrite(GroupByQuery query) { return Collections2.filter( query.getDimensions(), new Predicate<DimensionSpec>() { @Override public boolean apply(DimensionSpec input) { return input.getExtractionFn() != null && ExtractionFn.ExtractionType.ONE_TO_ONE.equals( input.getExtractionFn().getExtractionType() ); } } ); } }
@Override public void numDimensions(GroupByQuery query) { setDimension("numDimensions", String.valueOf(query.getDimensions().size())); }
public HashAggregateIterator( GroupByQuery query, GroupByQueryConfig querySpecificConfig, Cursor cursor, ByteBuffer buffer, DateTime fudgeTimestamp, GroupByColumnSelectorPlus[] dims, boolean allSingleValueDims ) { super(query, querySpecificConfig, cursor, buffer, fudgeTimestamp, dims, allSingleValueDims); final int dimCount = query.getDimensions().size(); stack = new int[dimCount]; valuess = new Object[dimCount]; keyBuffer = ByteBuffer.allocate(keySerde.keySize()); }
@Override public Comparable[] apply(Row row, Comparable[] key) { for (int i = 0; i < key.length; i++) { final Comparable val = (Comparable) row.getRaw(query.getDimensions().get(i).getOutputName()); key[i] = valueConvertFns[i].apply(val); } return key; } };
@Override public Comparable[] apply(Row row, Comparable[] key) { key[0] = timestampExtractFn.apply(row); for (int i = 1; i < key.length; i++) { final Comparable val = (Comparable) row.getRaw(query.getDimensions().get(i - 1).getOutputName()); key[i] = valueConvertFns[i - 1].apply(val); } return key; } };
public RowIterator(GroupByQuery query, Cursor cursor, ByteBuffer metricsBuffer, GroupByQueryConfig config) { this.query = query; this.cursor = cursor; this.metricsBuffer = metricsBuffer; this.config = config; unprocessedKeys = null; delegate = Iterators.emptyIterator(); dimensionSpecs = query.getDimensions(); dimensions = Lists.newArrayListWithExpectedSize(dimensionSpecs.size()); dimNames = Lists.newArrayListWithExpectedSize(dimensionSpecs.size()); for (int i = 0; i < dimensionSpecs.size(); ++i) { final DimensionSpec dimSpec = dimensionSpecs.get(i); final DimensionSelector selector = cursor.makeDimensionSelector(dimSpec.getDimension()); if (selector != null) { dimensions.add(selector); dimNames.add(dimSpec.getOutputName()); } } aggregatorSpecs = query.getAggregatorSpecs(); aggregators = new BufferAggregator[aggregatorSpecs.size()]; metricNames = new String[aggregatorSpecs.size()]; sizesRequired = new int[aggregatorSpecs.size()]; for (int i = 0; i < aggregatorSpecs.size(); ++i) { AggregatorFactory aggregatorSpec = aggregatorSpecs.get(i); aggregators[i] = aggregatorSpec.factorizeBuffered(cursor); metricNames[i] = aggregatorSpec.getName(); sizesRequired[i] = aggregatorSpec.getMaxIntermediateSize(); } }
for (DimensionSpec dimensionSpec : query.getDimensions()) { final String dimension = dimensionSpec.getOutputName(); if (optimizedDims.contains(dimension)) {
dimensionSpecs = query.getDimensions(); dimensions = Lists.newArrayListWithExpectedSize(dimensionSpecs.size()); dimNames = Lists.newArrayListWithExpectedSize(dimensionSpecs.size());
query.getDimensions(), new Function<DimensionSpec, String>()
@Override public ServiceMetricEvent.Builder makeMetricBuilder(GroupByQuery query) { int numMinutes = 0; for (Interval interval : query.getIntervals()) { numMinutes += Minutes.minutesIn(interval).getMinutes(); } return new ServiceMetricEvent.Builder() .setUser2(query.getDataSource().toString()) .setUser3(String.format("%,d dims", query.getDimensions().size())) .setUser4("groupBy") .setUser5(Joiner.on(",").join(query.getIntervals())) .setUser6(String.valueOf(query.hasFilters())) .setUser7(String.format("%,d aggs", query.getAggregatorSpecs().size())) .setUser9(Minutes.minutes(numMinutes).toString()); }
for (DimensionSpec dimensionSpec : query.getDimensions()) { types.put(dimensionSpec.getOutputName(), dimensionSpec.getOutputType());
@Override public Row apply(final Row arg1, final Row arg2) { if (arg1 == null) { return arg2; } else if (arg2 == null) { return arg1; } final Map<String, Object> newMap = Maps.newHashMapWithExpectedSize( query.getDimensions().size() + query.getAggregatorSpecs().size() ); // Add dimensions for (DimensionSpec dimension : query.getDimensions()) { newMap.put(dimension.getOutputName(), arg1.getRaw(dimension.getOutputName())); } // Add aggregations for (AggregatorFactory aggregatorFactory : query.getAggregatorSpecs()) { newMap.put( aggregatorFactory.getName(), aggregatorFactory.combine( arg1.getRaw(aggregatorFactory.getName()), arg2.getRaw(aggregatorFactory.getName()) ) ); } return new MapBasedRow(adjustTimestamp(arg1), newMap); }
public boolean determineApplyLimitPushDown() { final boolean forceLimitPushDown = validateAndGetForceLimitPushDown(); if (limitSpec instanceof DefaultLimitSpec) { DefaultLimitSpec defaultLimitSpec = (DefaultLimitSpec) limitSpec; // If only applying an orderby without a limit, don't try to push down if (!defaultLimitSpec.isLimited()) { return false; } if (forceLimitPushDown) { return true; } if (!getApplyLimitPushDownFromContext()) { return false; } if (havingSpec != null) { return false; } // If the sorting order only uses columns in the grouping key, we can always push the limit down // to the buffer grouper without affecting result accuracy boolean sortHasNonGroupingFields = DefaultLimitSpec.sortingOrderHasNonGroupingFields( (DefaultLimitSpec) limitSpec, getDimensions() ); return !sortHasNonGroupingFields; } return false; }
final Supplier<Comparable>[] inputRawSuppliers = getValueSuppliersForDimensions( columnSelectorFactory, query.getDimensions() );
for (DimensionSpec dimensionSpec : delegateGroupByQuery.getDimensions()) { if (optimizedDimensions.contains(dimensionSpec.getDimension())) { dimensionSpecs.add(
private CloseableGrouperIterator<KeyType, Row> initNewDelegate() { final Grouper<KeyType> grouper = newGrouper(); grouper.init(); if (allSingleValueDims) { aggregateSingleValueDims(grouper); } else { aggregateMultiValueDims(grouper); } return new CloseableGrouperIterator<>( grouper, false, entry -> { Map<String, Object> theMap = Maps.newLinkedHashMap(); // Add dimensions. putToMap(entry.getKey(), theMap); convertRowTypesToOutputTypes(query.getDimensions(), theMap); // Add aggregations. for (int i = 0; i < entry.getValues().length; i++) { theMap.put(query.getAggregatorSpecs().get(i).getName(), entry.getValues()[i]); } return new MapBasedRow(timestamp, theMap); }, grouper ); }
public Builder(GroupByQuery query) { dataSource = query.getDataSource(); querySegmentSpec = query.getQuerySegmentSpec(); virtualColumns = query.getVirtualColumns(); dimFilter = query.getDimFilter(); granularity = query.getGranularity(); dimensions = query.getDimensions(); aggregatorSpecs = query.getAggregatorSpecs(); postAggregatorSpecs = query.getPostAggregatorSpecs(); havingSpec = query.getHavingSpec(); limitSpec = query.getLimitSpec(); postProcessingFn = query.postProcessingFn; context = query.getContext(); }
for (DimensionSpec dimension : subquery.getDimensions()) { dimensionNames.add(dimension.getOutputName());
@Override public byte[] computeCacheKey(GroupByQuery query) { return new CacheKeyBuilder(GROUPBY_QUERY) .appendByte(CACHE_STRATEGY_VERSION) .appendCacheable(query.getGranularity()) .appendCacheable(query.getDimFilter()) .appendCacheables(query.getAggregatorSpecs()) .appendCacheables(query.getDimensions()) .appendCacheable(query.getVirtualColumns()) .build(); }
query.getDimFilter(), query.getGranularity(), query.getDimensions(), query.getAggregatorSpecs(), query.getPostAggregatorSpecs(),