public void applyLimitPushDown(IRealization realization, StorageLimitLevel storageLimitLevel) { if (storageLimitLevel == StorageLimitLevel.NO_LIMIT) { return; } if (!realization.supportsLimitPushDown()) { logger.warn("Not enabling limit push down because cube storage type not supported"); return; } long temp = this.getOffset() + this.getLimit(); if (!isValidPushDownLimit(temp)) { logger.warn("Not enabling limit push down because current limit is invalid: " + this.getLimit()); return; } this.finalPushDownLimit = (int) temp; this.storageLimitLevel = storageLimitLevel; logger.info("Enabling limit push down: {} at level: {}", temp, storageLimitLevel); }
public SerializedHBaseTupleIterator(HConnection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, Collection<TblColRef> dimensions, TupleFilter filter, Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context) { this.context = context; int limit = context.getLimit(); this.partialResultLimit = Math.max(limit, PARTIAL_DEFAULT_LIMIT); this.segmentIteratorList = new ArrayList<CubeSegmentTupleIterator>(segmentKeyRanges.size()); Map<CubeSegment, List<HBaseKeyRange>> rangesMap = makeRangesMap(segmentKeyRanges); for (Map.Entry<CubeSegment, List<HBaseKeyRange>> entry : rangesMap.entrySet()) { CubeSegmentTupleIterator segIter = new CubeSegmentTupleIterator(entry.getKey(), entry.getValue(), conn, dimensions, filter, groupBy, rowValueDecoders, context); this.segmentIteratorList.add(segIter); } this.segmentIteratorIterator = this.segmentIteratorList.iterator(); if (this.segmentIteratorIterator.hasNext()) { this.segmentIterator = this.segmentIteratorIterator.next(); } else { this.segmentIterator = ITupleIterator.EMPTY_TUPLE_ITERATOR; } }
private void setLimit(TupleFilter filter, StorageContext context) { boolean goodAggr = context.isExactAggregation(); boolean goodFilter = filter == null || (TupleFilter.isEvaluableRecursively(filter) && context.isCoprocessorEnabled()); boolean goodSort = context.hasSort() == false; if (goodAggr && goodFilter && goodSort) { logger.info("Enable limit " + context.getLimit()); context.enableLimit(); } }
@Override public boolean hasNext() { // 1. check limit if (context.isLimitEnabled() && scanCount >= context.getLimit()) { return false; } // 2. check partial result if (context.isAcceptPartialResult() && scanCount > partialResultLimit) { context.setPartialResultReturned(true); return false; } // 3. check threshold if (scanCount >= context.getThreshold()) { throw new ScanOutOfLimitException("Scan row count exceeded threshold: " + context.getThreshold() + ", please add filter condition to narrow down backend scan range, like where clause."); } // 4. check cube segments return segmentIteratorIterator.hasNext() || segmentIterator.hasNext(); }
public void applyLimitPushDown(IRealization realization, StorageLimitLevel storageLimitLevel) { if (storageLimitLevel == StorageLimitLevel.NO_LIMIT) { return; } if (!realization.supportsLimitPushDown()) { logger.warn("Not enabling limit push down because cube storage type not supported"); return; } long temp = this.getOffset() + this.getLimit(); if (!isValidPushDownLimit(temp)) { logger.warn("Not enabling limit push down because current limit is invalid: " + this.getLimit()); return; } this.finalPushDownLimit = (int) temp; this.storageLimitLevel = storageLimitLevel; logger.info("Enabling limit push down: {} at level: {}", temp, storageLimitLevel); }