public void aggregate(Object[] values, int[] measures) { assert values.length == descLength; for (int i = 0; i < measures.length; i++) { aggs[measures[i]].aggregate(values[measures[i]]); } }
public static long estimateSizeOf(MeasureAggregator[] aggrs) { // size of array, AggregationCacheMemSizeTest reports 4 for [0], 12 for [1], 12 for [2], 20 for [3] etc.. // Memory alignment to 8 bytes long est = (aggrs.length + 1) / 2 * 8L + 4 + (4 /* extra */); for (MeasureAggregator aggr : aggrs) { if (aggr != null) est += aggr.getMemBytesEstimate(); } return est; }
public void collectStates(Object[] states) { for (int i = 0; i < descLength; i++) { states[i] = aggs[i].getState(); } } }
@Test public void testAggregatorEstimate() { HLLCAggregator hllcAggregator = new HLLCAggregator(14); hllcAggregator.aggregate(new HLLCounter(14)); BitmapAggregator bitmapAggregator = new BitmapAggregator(); BitmapCounter bitmapCounter = RoaringBitmapCounterFactory.INSTANCE.newBitmap(); for (int i = 4000; i <= 100000; i += 2) { bitmapCounter.add(i); } bitmapAggregator.aggregate(bitmapCounter); ExtendedColumnMeasureType extendedColumnType = new ExtendedColumnMeasureType( DataType.getType("extendedcolumn(100)")); MeasureAggregator<ByteArray> extendedColumnAggregator = extendedColumnType.newAggregator(); extendedColumnAggregator.aggregate(new ByteArray(100)); List<MeasureAggregator> aggregators = Lists.newArrayList(basicAggregators()); aggregators.add(hllcAggregator); aggregators.add(bitmapAggregator); aggregators.add(extendedColumnAggregator); System.out.printf(Locale.ROOT, "%40s %10s %10s\n", "Class", "Estimate", "Actual"); for (MeasureAggregator aggregator : aggregators) { String clzName = getAggregatorName(aggregator.getClass()); System.out.printf(Locale.ROOT, "%40s %10d %10d\n", clzName, aggregator.getMemBytesEstimate(), meter.measureDeep(aggregator)); } }
@Override public MeasureAggregator<?>[] newMetricsAggregators(ImmutableBitSet columns, String[] aggrFunctions) { assert columns.trueBitCount() == aggrFunctions.length; MeasureAggregator<?>[] result = new MeasureAggregator[aggrFunctions.length]; for (int i = 0; i < result.length; i++) { int col = columns.trueBitAt(i); result[i] = MeasureAggregator.create(aggrFunctions[i], info.getColumnType(col)); } // deal with holistic distinct count if (dependentMetricsMap != null) { for (Map.Entry<Integer, Integer> childEntry : dependentMetricsMap.entrySet()) { if (columns.get(childEntry.getKey())) { Integer parent = childEntry.getValue(); if (columns.get(parent) == false) throw new IllegalStateException(); int childIdx = columns.trueBitIndexOf(childEntry.getKey()); int parentIdx = columns.trueBitIndexOf(parent); result[childIdx].setDependentAggregator(result[parentIdx]); } } } return result; }
public MeasureAggregators(MeasureDesc... measureDescs) { descLength = measureDescs.length; aggs = new MeasureAggregator[descLength]; Map<String, Integer> measureIndexMap = new HashMap<String, Integer>(); for (int i = 0; i < descLength; i++) { FunctionDesc func = measureDescs[i].getFunction(); aggs[i] = func.getMeasureType().newAggregator(); measureIndexMap.put(measureDescs[i].getName(), i); } // fill back dependent aggregator for (int i = 0; i < descLength; i++) { String depMsrRef = measureDescs[i].getDependentMeasureRef(); if (depMsrRef != null) { int index = measureIndexMap.get(depMsrRef); aggs[i].setDependentAggregator(aggs[index]); } } }
public void reset() { for (int i = 0; i < aggs.length; i++) { aggs[i].reset(); } }
@Override public MeasureAggregator<?>[] newMetricsAggregators(ImmutableBitSet columns, String[] aggrFunctions) { assert columns.trueBitCount() == aggrFunctions.length; MeasureAggregator<?>[] result = new MeasureAggregator[aggrFunctions.length]; for (int i = 0; i < result.length; i++) { int col = columns.trueBitAt(i); result[i] = MeasureAggregator.create(aggrFunctions[i], info.getColumnType(col)); } return result; }
@Override public MeasureAggregator<?>[] newMetricsAggregators(ImmutableBitSet columns, String[] aggrFunctions) { assert columns.trueBitCount() == aggrFunctions.length; MeasureAggregator<?>[] result = new MeasureAggregator[aggrFunctions.length]; for (int i = 0; i < result.length; i++) { int col = columns.trueBitAt(i); result[i] = MeasureAggregator.create(aggrFunctions[i], info.getColumnType(col)); } // deal with holistic distinct count if (dependentMetricsMap != null) { for (Map.Entry<Integer, Integer> childEntry : dependentMetricsMap.entrySet()) { if (columns.get(childEntry.getKey())) { Integer parent = childEntry.getValue(); if (columns.get(parent) == false) throw new IllegalStateException(); int childIdx = columns.trueBitIndexOf(childEntry.getKey()); int parentIdx = columns.trueBitIndexOf(parent); result[childIdx].setDependentAggregator(result[parentIdx]); } } } return result; }
public MeasureAggregators(MeasureDesc... measureDescs) { descLength = measureDescs.length; aggs = new MeasureAggregator[descLength]; Map<String, Integer> measureIndexMap = new HashMap<String, Integer>(); for (int i = 0; i < descLength; i++) { FunctionDesc func = measureDescs[i].getFunction(); aggs[i] = func.getMeasureType().newAggregator(); measureIndexMap.put(measureDescs[i].getName(), i); } // fill back dependent aggregator for (int i = 0; i < descLength; i++) { String depMsrRef = measureDescs[i].getDependentMeasureRef(); if (depMsrRef != null) { int index = measureIndexMap.get(depMsrRef); aggs[i].setDependentAggregator(aggs[index]); } } }
public void reset() { for (int i = 0; i < aggs.length; i++) { aggs[i].reset(); } }
@Override public MeasureAggregator<?>[] newMetricsAggregators(ImmutableBitSet columns, String[] aggrFunctions) { assert columns.trueBitCount() == aggrFunctions.length; MeasureAggregator<?>[] result = new MeasureAggregator[aggrFunctions.length]; for (int i = 0; i < result.length; i++) { int col = columns.trueBitAt(i); result[i] = MeasureAggregator.create(aggrFunctions[i], info.getColumnType(col)); } return result; }
public void aggregate(Object[] values1, Object[] values2, Object[] result) { assert values1.length == values2.length && values2.length == descLength && values1.length == result.length; for (int i = 0; i < descLength; i++) { result[i] = aggs[i].aggregate(values1[i], values2[i]); } }
@Override public Object getValue(TblColRef col) { return aggrValues[col.getColumnDesc().getZeroBasedIndex()].getState(); }
public void checkMemoryUsage() { if (firstKey == null) return; // about memory calculation, // http://seniorjava.wordpress.com/2013/09/01/java-objects-memory-size-reference/ if (rowMemBytes <= 0) { if (aggBufMap.size() > 0) { rowMemBytes = 0; MeasureAggregator[] measureAggregators = aggBufMap.get(firstKey); for (MeasureAggregator agg : measureAggregators) { rowMemBytes += agg.getMemBytesEstimate(); } } } int size = aggBufMap.size(); long memUsage = (40L + rowMemBytes) * size; if (memUsage > MEMORY_USAGE_CAP) { throw new RuntimeException("Kylin coprocessor memory usage goes beyond cap, (40 + " + rowMemBytes + ") * " + size + " > " + MEMORY_USAGE_CAP + ". Abort coprocessor."); } //If less than 5% of max memory long avail = MemoryBudgetController.getSystemAvailBytes(); if (avail < (MEMOERY_MAX_BYTES / 20)) { throw new RuntimeException("Running Kylin coprocessor when too little memory is left. Abort coprocessor. Current available memory is " + avail + ". Max memory is " + MEMOERY_MAX_BYTES); } } }
public MeasureAggregators(MeasureDesc... measureDescs) { descLength = measureDescs.length; aggs = new MeasureAggregator[descLength]; Map<String, Integer> measureIndexMap = new HashMap<String, Integer>(); for (int i = 0; i < descLength; i++) { FunctionDesc func = measureDescs[i].getFunction(); aggs[i] = func.getMeasureType().newAggregator(); measureIndexMap.put(measureDescs[i].getName(), i); } // fill back dependent aggregator for (int i = 0; i < descLength; i++) { String depMsrRef = measureDescs[i].getDependentMeasureRef(); if (depMsrRef != null) { int index = measureIndexMap.get(depMsrRef); aggs[i].setDependentAggregator(aggs[index]); } } }
public void reset() { for (int i = 0; i < aggs.length; i++) { aggs[i].reset(); } }
public void aggregate(Object[] values) { assert values.length == descLength; for (int i = 0; i < descLength; i++) { aggs[i].aggregate(values[i]); } }
void load(byte[] key, MeasureAggregator[] value) { int offset = 0; for (int i = 0; i < dimensions.trueBitCount(); i++) { int c = dimensions.trueBitAt(i); final int columnLength = info.codeSystem.maxCodeLength(c); record.cols[c].reset(key, offset, columnLength); offset += columnLength; } for (int i = 0; i < value.length; i++) { tmpValues[i] = value[i].getState(); } byte[] bytes = measureCodec.encode(tmpValues).array(); int[] sizes = measureCodec.getMeasureSizes(); offset = 0; for (int i = 0; i < value.length; i++) { int col = metrics.trueBitAt(i); record.cols[col].reset(bytes, offset, sizes[i]); offset += sizes[i]; } } }
public static long estimateSizeOf(MeasureAggregator[] aggrs) { // size of array, AggregationCacheMemSizeTest reports 4 for [0], 12 for [1], 12 for [2], 20 for [3] etc.. // Memory alignment to 8 bytes long est = (aggrs.length + 1) / 2 * 8L + 4 + (4 /* extra */); for (MeasureAggregator aggr : aggrs) { if (aggr != null) est += aggr.getMemBytesEstimate(); } return est; }