try { ByteBuffer out = ByteBuffer.allocate(scanRequestBufferSize); GTInfo.serializer.serialize(scanRequest.getInfo(), out); BytesUtil.writeVInt(scanRequest.getGTScanRanges().size(), out); for (GTScanRange range : scanRequest.getGTScanRanges()) { serializeGTRecord(range.pkStart, out); serializeGTRecord(range.pkEnd, out); ImmutableBitSet.serializer.serialize(scanRequest.getColumns(), out); BytesUtil.writeByteArray( GTUtil.serializeGTFilter(scanRequest.getFilterPushDown(), scanRequest.getInfo()), out); ImmutableBitSet.serializer.serialize(scanRequest.getAggrGroupBy(), out); ImmutableBitSet.serializer.serialize(scanRequest.getAggrMetrics(), out); BytesUtil.writeAsciiStringArray(scanRequest.getAggrMetricsFuncs(), out); BytesUtil.writeVInt(scanRequest.isAllowStorageAggregation() ? 1 : 0, out); BytesUtil.writeUTFString(scanRequest.getStorageLimitLevel().name(), out); BytesUtil.writeVInt(scanRequest.getStorageScanRowNumThreshold(), out); BytesUtil.writeVInt(scanRequest.getStoragePushDownLimit(), out); BytesUtil.writeUTFString(scanRequest.getStorageBehavior(), out); out.flip(); return Bytes.toStringBinary(out.array(), out.position(), out.limit());
/** * filterToggledOn,aggrToggledOn are only for profiling/test use. * in normal cases they are all true. * * Refer to CoprocessorBehavior for explanation */ public IGTScanner decorateScanner(IGTScanner scanner, boolean filterToggledOn, boolean aggrToggledOn) throws IOException { return decorateScanner(scanner, filterToggledOn, aggrToggledOn, false, true); }
public GTAggregateScanner(IGTScanner input, GTScanRequest req, boolean spillEnabled) { if (!req.hasAggregation()) throw new IllegalStateException(); if (input instanceof GTFilterScanner) { logger.info("setting IGTBypassChecker of child"); ((GTFilterScanner) input).setChecker(this); } else { logger.info("applying a GTFilterScanner with IGTBypassChecker on top child"); input = new GTFilterScanner(input, null, this); } this.inputScanner = input; this.info = this.inputScanner.getInfo(); this.dimensions = req.getDimensions(); this.groupBy = req.getAggrGroupBy(); this.metrics = req.getAggrMetrics(); this.metricsAggrFuncs = req.getAggrMetricsFuncs(); this.measureCodec = req.createMeasureCodec(); this.spillThreshold = (long) (req.getAggCacheMemThreshold() * MemoryBudgetController.ONE_GB); this.aggrMask = new boolean[metricsAggrFuncs.length]; this.storagePushDownLimit = req.getStoragePushDownLimit(); this.storageLimitLevel = req.getStorageLimitLevel(); this.spillEnabled = spillEnabled; this.havingFilter = req.getHavingFilterPushDown(); this.aggrCache = new AggregationCache(); Arrays.fill(aggrMask, true); }
AbstractStreamMergeIterator(Iterator<GTRecord> input) { this.input = Iterators.peekingIterator(input); this.codeSystem = req.getInfo().getCodeSystem(); this.dimensions = req.getDimensions(); this.metrics = req.getAggrMetrics(); this.metricFuncs = req.getAggrMetricsFuncs(); this.measureCodec = req.createMeasureCodec(); this.first = new GTRecord(req.getInfo()); }
public StorageResponseGTScatter(GTScanRequest scanRequest, IPartitionStreamer partitionStreamer, StorageContext context) { this.info = scanRequest.getInfo(); this.partitionStreamer = partitionStreamer; this.blocks = partitionStreamer.asByteArrayIterator(); this.columns = scanRequest.getColumns(); this.groupByDims = scanRequest.getAggrGroupBy(); this.needSorted = (context.getFinalPushDownLimit() != Integer.MAX_VALUE) || context.isStreamAggregateEnabled(); }
final long deadline = scanReq.getStartTime() + scanReq.getTimeout(); checkDeadline(deadline); StorageSideBehavior behavior = StorageSideBehavior.valueOf(scanReq.getStorageBehavior()); final List<RawScan> hbaseRawScans = deserializeRawScans( ByteBuffer.wrap(HBaseZeroCopyByteString.zeroCopyGetBytes(request.getHbaseRawScan()))); appendProfileInfo(sb, "start latency: " + (serviceStartTime - scanReq.getStartTime()), serviceStartTime); scanReq.disableAggCacheMemCheck(); // disable mem check if so told final long storagePushDownLimit = scanReq.getStoragePushDownLimit(); scanReq.getStorageScanRowNumThreshold(), // for old client (scan threshold) IGTScanner finalScanner = scanReq.decorateScanner(rawScanner, behavior.filterToggledOn(), behavior.aggrToggledOn(), false, request.getSpillEnabled()); buffer.clear(); try { oneRecord.exportColumns(scanReq.getColumns(), buffer); } catch (BufferOverflowException boe) { buffer = ByteBuffer.allocate(oneRecord.sizeOf(scanReq.getColumns()) * 2); oneRecord.exportColumns(scanReq.getColumns(), buffer); if (!scanReq.isDoingStorageAggregation() && (scanReq.getStorageLimitLevel() != StorageLimitLevel.NO_LIMIT && finalRowCount >= storagePushDownLimit)) {
/** * @param withDelay is for test use */ public HBaseReadonlyStore(CellListIterator cellListIterator, GTScanRequest gtScanRequest, List<Pair<byte[], byte[]>> hbaseColumns, List<List<Integer>> hbaseColumnsToGT, int rowkeyPreambleSize, boolean withDelay, boolean isExactAggregation) { this.cellListIterator = cellListIterator; this.info = gtScanRequest.getInfo(); this.hbaseColumns = hbaseColumns; this.hbaseColumnsToGT = hbaseColumnsToGT; this.rowkeyPreambleSize = rowkeyPreambleSize; this.withDelay = withDelay; this.isExactAggregation = isExactAggregation; }
public static GTScanRequest useDeserializedGTScanRequest(GTScanRequest origin) { ByteBuffer buffer = ByteBuffer.allocate(BytesSerializer.SERIALIZE_BUFFER_SIZE); GTScanRequest.serializer.serialize(origin, buffer); buffer.flip(); GTScanRequest sGTScanRequest = GTScanRequest.serializer.deserialize(buffer); Assert.assertArrayEquals(origin.getAggrMetricsFuncs(), sGTScanRequest.getAggrMetricsFuncs()); Assert.assertEquals(origin.getAggCacheMemThreshold(), sGTScanRequest.getAggCacheMemThreshold(), 0.01); return sGTScanRequest; }
private IGTScanner getGTScannerInternal(final GTScanRequest scanRequest) throws IOException { final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0); List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks); List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks); IGTScanner rawScanner = store.scan(scanRequest); final IGTScanner decorateScanner = scanRequest.decorateScanner(rawScanner); final TrimmedInfoGTRecordAdapter trimmedInfoGTRecordAdapter = new TrimmedInfoGTRecordAdapter(fullGTInfo, decorateScanner.iterator());
public GTStreamAggregateScanner(IGTScanner delegated, GTScanRequest scanRequest) { super(delegated); this.req = Preconditions.checkNotNull(scanRequest, "scanRequest"); this.keyComparator = GTRecord.getComparator(scanRequest.getAggrGroupBy()); }
public GTFilterScanner(IGTScanner delegated, GTScanRequest req, IGTBypassChecker checker) { super(delegated); this.checker = checker; if (req != null) { this.filter = req.getFilterPushDown(); this.filterCodeSystem = GTUtil.wrap(getInfo().codeSystem.getComparator()); this.oneTuple = new IEvaluatableTuple() { @Override public Object getValue(TblColRef col) { return next.get(col.getColumnDesc().getZeroBasedIndex()); } }; if (!TupleFilter.isEvaluableRecursively(filter)) throw new IllegalArgumentException(); } }
BytesUtil.writeVInt(value.allowStorageAggregation ? 1 : 0, out); out.putDouble(value.aggCacheMemThreshold); BytesUtil.writeUTFString(value.getStorageLimitLevel().name(), out); BytesUtil.writeVInt(value.storageScanRowNumThreshold, out); BytesUtil.writeVInt(value.storagePushDownLimit, out);
final long deadline = scanReq.getStartTime() + scanReq.getTimeout(); checkDeadline(deadline); StorageSideBehavior behavior = StorageSideBehavior.valueOf(scanReq.getStorageBehavior()); final List<RawScan> hbaseRawScans = deserializeRawScans( ByteBuffer.wrap(HBaseZeroCopyByteString.zeroCopyGetBytes(request.getHbaseRawScan()))); appendProfileInfo(sb, "start latency: " + (serviceStartTime - scanReq.getStartTime()), serviceStartTime); scanReq.disableAggCacheMemCheck(); // disable mem check if so told final long storagePushDownLimit = scanReq.getStoragePushDownLimit(); scanReq.getStorageScanRowNumThreshold(), // for old client (scan threshold) IGTScanner finalScanner = scanReq.decorateScanner(rawScanner, behavior.filterToggledOn(), behavior.aggrToggledOn(), false, request.getSpillEnabled()); buffer.clear(); try { oneRecord.exportColumns(scanReq.getColumns(), buffer); } catch (BufferOverflowException boe) { buffer = ByteBuffer.allocate(oneRecord.sizeOf(scanReq.getColumns()) * 2); oneRecord.exportColumns(scanReq.getColumns(), buffer); if (!scanReq.isDoingStorageAggregation() && (scanReq.getStorageLimitLevel() != StorageLimitLevel.NO_LIMIT && finalRowCount >= storagePushDownLimit)) {
AbstractStreamMergeIterator(Iterator<GTRecord> input) { this.input = Iterators.peekingIterator(input); this.codeSystem = req.getInfo().getCodeSystem(); this.dimensions = req.getDimensions(); this.metrics = req.getAggrMetrics(); this.metricFuncs = req.getAggrMetricsFuncs(); this.measureCodec = req.createMeasureCodec(); this.first = new GTRecord(req.getInfo()); }
public StorageResponseGTScatter(GTScanRequest scanRequest, IPartitionStreamer partitionStreamer, StorageContext context) { this.info = scanRequest.getInfo(); this.partitionStreamer = partitionStreamer; this.blocks = partitionStreamer.asByteArrayIterator(); this.columns = scanRequest.getColumns(); this.groupByDims = scanRequest.getAggrGroupBy(); this.needSorted = (context.getFinalPushDownLimit() != Integer.MAX_VALUE) || context.isStreamAggregateEnabled(); }
public GTInfo getInfo() { return scanRequest == null ? null : scanRequest.getInfo(); }
private IGTScanner getGTScannerInternal(final GTScanRequest scanRequest) throws IOException { final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0); List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks); List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks); IGTScanner rawScanner = store.scan(scanRequest); final IGTScanner decorateScanner = scanRequest.decorateScanner(rawScanner); final TrimmedInfoGTRecordAdapter trimmedInfoGTRecordAdapter = new TrimmedInfoGTRecordAdapter(fullGTInfo, decorateScanner.iterator());
public GTStreamAggregateScanner(IGTScanner delegated, GTScanRequest scanRequest) { super(delegated); this.req = Preconditions.checkNotNull(scanRequest, "scanRequest"); this.keyComparator = GTRecord.getComparator(scanRequest.getAggrGroupBy()); }
public GTFilterScanner(IGTScanner delegated, GTScanRequest req, IGTBypassChecker checker) { super(delegated); this.checker = checker; if (req != null) { this.filter = req.getFilterPushDown(); this.filterCodeSystem = GTUtil.wrap(getInfo().codeSystem.getComparator()); this.oneTuple = new IEvaluatableTuple() { @Override public Object getValue(TblColRef col) { return next.get(col.getColumnDesc().getZeroBasedIndex()); } }; if (!TupleFilter.isEvaluableRecursively(filter)) throw new IllegalArgumentException(); } }
BytesUtil.writeVInt(value.allowStorageAggregation ? 1 : 0, out); out.putDouble(value.aggCacheMemThreshold); BytesUtil.writeUTFString(value.getStorageLimitLevel().name(), out); BytesUtil.writeVInt(value.storageScanRowNumThreshold, out); BytesUtil.writeVInt(value.storagePushDownLimit, out);