public static long estimateSizeOf(MeasureAggregator[] aggrs) { // size of array, AggregationCacheMemSizeTest reports 4 for [0], 12 for [1], 12 for [2], 20 for [3] etc.. // Memory alignment to 8 bytes long est = (aggrs.length + 1) / 2 * 8L + 4 + (4 /* extra */); for (MeasureAggregator aggr : aggrs) { if (aggr != null) est += aggr.getMemBytesEstimate(); } return est; }
public void checkMemoryUsage() { if (firstKey == null) return; // about memory calculation, // http://seniorjava.wordpress.com/2013/09/01/java-objects-memory-size-reference/ if (rowMemBytes <= 0) { if (aggBufMap.size() > 0) { rowMemBytes = 0; MeasureAggregator[] measureAggregators = aggBufMap.get(firstKey); for (MeasureAggregator agg : measureAggregators) { rowMemBytes += agg.getMemBytesEstimate(); } } } int size = aggBufMap.size(); long memUsage = (40L + rowMemBytes) * size; if (memUsage > MEMORY_USAGE_CAP) { throw new RuntimeException("Kylin coprocessor memory usage goes beyond cap, (40 + " + rowMemBytes + ") * " + size + " > " + MEMORY_USAGE_CAP + ". Abort coprocessor."); } //If less than 5% of max memory long avail = MemoryBudgetController.getSystemAvailBytes(); if (avail < (MEMOERY_MAX_BYTES / 20)) { throw new RuntimeException("Running Kylin coprocessor when too little memory is left. Abort coprocessor. Current available memory is " + avail + ". Max memory is " + MEMOERY_MAX_BYTES); } } }
@Test public void testAggregatorEstimate() { HLLCAggregator hllcAggregator = new HLLCAggregator(14); hllcAggregator.aggregate(new HLLCounter(14)); BitmapAggregator bitmapAggregator = new BitmapAggregator(); BitmapCounter bitmapCounter = RoaringBitmapCounterFactory.INSTANCE.newBitmap(); for (int i = 4000; i <= 100000; i += 2) { bitmapCounter.add(i); } bitmapAggregator.aggregate(bitmapCounter); ExtendedColumnMeasureType extendedColumnType = new ExtendedColumnMeasureType( DataType.getType("extendedcolumn(100)")); MeasureAggregator<ByteArray> extendedColumnAggregator = extendedColumnType.newAggregator(); extendedColumnAggregator.aggregate(new ByteArray(100)); List<MeasureAggregator> aggregators = Lists.newArrayList(basicAggregators()); aggregators.add(hllcAggregator); aggregators.add(bitmapAggregator); aggregators.add(extendedColumnAggregator); System.out.printf(Locale.ROOT, "%40s %10s %10s\n", "Class", "Estimate", "Actual"); for (MeasureAggregator aggregator : aggregators) { String clzName = getAggregatorName(aggregator.getClass()); System.out.printf(Locale.ROOT, "%40s %10d %10d\n", clzName, aggregator.getMemBytesEstimate(), meter.measureDeep(aggregator)); } }
public static long estimateSizeOf(MeasureAggregator[] aggrs) { // size of array, AggregationCacheMemSizeTest reports 4 for [0], 12 for [1], 12 for [2], 20 for [3] etc.. // Memory alignment to 8 bytes long est = (aggrs.length + 1) / 2 * 8L + 4 + (4 /* extra */); for (MeasureAggregator aggr : aggrs) { if (aggr != null) est += aggr.getMemBytesEstimate(); } return est; }
public void checkMemoryUsage() { if (firstKey == null) return; // about memory calculation, // http://seniorjava.wordpress.com/2013/09/01/java-objects-memory-size-reference/ if (rowMemBytes <= 0) { if (aggBufMap.size() > 0) { rowMemBytes = 0; MeasureAggregator[] measureAggregators = aggBufMap.get(firstKey); for (MeasureAggregator agg : measureAggregators) { rowMemBytes += agg.getMemBytesEstimate(); } } } int size = aggBufMap.size(); long memUsage = (40L + rowMemBytes) * size; if (memUsage > MEMORY_USAGE_CAP) { throw new RuntimeException("Kylin coprocessor memory usage goes beyond cap, (40 + " + rowMemBytes + ") * " + size + " > " + MEMORY_USAGE_CAP + ". Abort coprocessor."); } //If less than 5% of max memory long avail = MemoryBudgetController.getSystemAvailBytes(); if (avail < (MEMOERY_MAX_BYTES / 20)) { throw new RuntimeException("Running Kylin coprocessor when too little memory is left. Abort coprocessor. Current available memory is " + avail + ". Max memory is " + MEMOERY_MAX_BYTES); } } }