private <T> CuboidResult createBaseCuboid(RecordConsumeBlockingQueueController<T> input) throws IOException { long startTime = System.currentTimeMillis(); logger.info("Calculating base cuboid {}", baseCuboidId); GridTable baseCuboid = newGridTableByCuboidID(baseCuboidId); GTBuilder baseBuilder = baseCuboid.rebuild(); IGTScanner baseInput = new InputConverter(baseCuboid.getInfo(), input); Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount); GTScanRequest req = new GTScanRequestBuilder().setInfo(baseCuboid.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(dimensionMetricsBitSet.getFirst()).setAggrMetrics(dimensionMetricsBitSet.getSecond()).setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req); aggregationScanner.trackMemoryLevel(baseCuboidMemTracker); int count = 0; try { for (GTRecord r : aggregationScanner) { if (count == 0) { baseCuboidMemTracker.markHigh(); } baseBuilder.write(r); count++; } } finally { aggregationScanner.close(); baseBuilder.close(); } long timeSpent = System.currentTimeMillis() - startTime; logger.info("Cuboid {} has {} rows, build takes {}ms", baseCuboidId, count, timeSpent); int mbEstimateBaseAggrCache = (int) (aggregationScanner.getEstimateSizeOfAggrCache() / MemoryBudgetController.ONE_MB); logger.info("Wild estimate of base aggr cache is {} MB", mbEstimateBaseAggrCache); return updateCuboidResult(baseCuboidId, baseCuboid, count, timeSpent, 0, input.inputConverterUnit.ifChange()); }
.setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); try (GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req)) { aggregationScanner.trackMemoryLevel(baseCuboidMemTracker);
private <T> CuboidResult createBaseCuboid(RecordConsumeBlockingQueueController<T> input) throws IOException { long startTime = System.currentTimeMillis(); logger.info("Calculating base cuboid {}", baseCuboidId); GridTable baseCuboid = newGridTableByCuboidID(baseCuboidId); GTBuilder baseBuilder = baseCuboid.rebuild(); IGTScanner baseInput = new InputConverter(baseCuboid.getInfo(), input); Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount); GTScanRequest req = new GTScanRequestBuilder().setInfo(baseCuboid.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(dimensionMetricsBitSet.getFirst()).setAggrMetrics(dimensionMetricsBitSet.getSecond()).setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req); aggregationScanner.trackMemoryLevel(baseCuboidMemTracker); int count = 0; try { for (GTRecord r : aggregationScanner) { if (count == 0) { baseCuboidMemTracker.markHigh(); } baseBuilder.write(r); count++; } } finally { aggregationScanner.close(); baseBuilder.close(); } long timeSpent = System.currentTimeMillis() - startTime; logger.info("Cuboid {} has {} rows, build takes {}ms", baseCuboidId, count, timeSpent); int mbEstimateBaseAggrCache = (int) (aggregationScanner.getEstimateSizeOfAggrCache() / MemoryBudgetController.ONE_MB); logger.info("Wild estimate of base aggr cache is {} MB", mbEstimateBaseAggrCache); return updateCuboidResult(baseCuboidId, baseCuboid, count, timeSpent, 0, input.inputConverterUnit.ifChange()); }
.setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); try (GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req)) { aggregationScanner.trackMemoryLevel(baseCuboidMemTracker);