public void collectStates(Object[] states) { for (int i = 0; i < descs.length; i++) { states[i] = aggs[i].getState(); } }
public void aggregate(Object[] values) { assert values.length == descs.length; for (int i = 0; i < descs.length; i++) { aggs[i].aggregate(values[i]); } }
public MeasureAggregator[] createBuffer() { MeasureAggregator[] aggrs = new MeasureAggregator[funcNames.length]; for (int i = 0; i < aggrs.length; i++) { if (metricInfos[i].type == MetricType.DistinctCount) { aggrs[i] = MeasureAggregator.create(funcNames[i], dataTypes[i]); } else { //all other fixed length measures can be aggregated as long aggrs[i] = MeasureAggregator.create(funcNames[i], "long"); } } return aggrs; }
public static MeasureAggregator<?> create(String funcName, String returnType) { if (FunctionDesc.FUNC_SUM.equalsIgnoreCase(funcName) || FunctionDesc.FUNC_COUNT.equalsIgnoreCase(funcName)) { if (isInteger(returnType)) return new LongSumAggregator(); else if (isBigDecimal(returnType)) return new BigDecimalSumAggregator(); else if (isDouble(returnType)) return new DoubleSumAggregator(); } else if (FunctionDesc.FUNC_COUNT_DISTINCT.equalsIgnoreCase(funcName)) { if (DataType.getInstance(returnType).isHLLC()) return new HLLCAggregator(); else return new LDCAggregator(); } else if (FunctionDesc.FUNC_MAX.equalsIgnoreCase(funcName)) { if (isInteger(returnType)) return new LongMaxAggregator(); else if (isBigDecimal(returnType)) return new BigDecimalMaxAggregator(); else if (isDouble(returnType)) return new DoubleMaxAggregator(); } else if (FunctionDesc.FUNC_MIN.equalsIgnoreCase(funcName)) { if (isInteger(returnType)) return new LongMinAggregator(); else if (isBigDecimal(returnType)) return new BigDecimalMinAggregator(); else if (isDouble(returnType)) return new DoubleMinAggregator(); } throw new IllegalArgumentException("No aggregator for func '" + funcName + "' and return type '" + returnType + "'"); }
public MeasureAggregators(MeasureDesc... measureDescs) { descs = measureDescs; aggs = new MeasureAggregator[descs.length]; Map<String, Integer> measureIndexMap = new HashMap<String, Integer>(); for (int i = 0; i < descs.length; i++) { FunctionDesc func = descs[i].getFunction(); aggs[i] = MeasureAggregator.create(func.getExpression(), func.getReturnType()); measureIndexMap.put(descs[i].getName(), i); } // fill back dependent aggregator for (int i = 0; i < descs.length; i++) { String depMsrRef = descs[i].getDependentMeasureRef(); if (depMsrRef != null) { int index = measureIndexMap.get(depMsrRef); aggs[i].setDependentAggregator(aggs[index]); } } }
public void reset() { for (int i = 0; i < aggs.length; i++) { aggs[i].reset(); } }
public void checkMemoryUsage() { // about memory calculation, // http://seniorjava.wordpress.com/2013/09/01/java-objects-memory-size-reference/ if (rowMemBytes <= 0) { if (aggBufMap.size() > 0) { rowMemBytes = 0; MeasureAggregator[] measureAggregators = aggBufMap.get(aggBufMap.firstKey()); for (MeasureAggregator agg : measureAggregators) { rowMemBytes += agg.getMemBytes(); } } } int size = aggBufMap.size(); int memUsage = (40 + rowMemBytes) * size; if (memUsage > MEMORY_USAGE_CAP) { throw new RuntimeException("Kylin coprocess memory usage goes beyond cap, (40 + " + rowMemBytes + ") * " + size + " > " + MEMORY_USAGE_CAP + ". Abord coprocessor."); } } }
public ByteBuffer[] getHColValues(MeasureAggregator[] aggrs) { int i = 0; for (int ci = 0; ci < nHCols; ci++) { HCol col = hcols[ci]; for (int j = 0; j < col.nMeasures; j++) col.measureValues[j] = aggrs[i++].getState(); col.measureBuf.clear(); col.measureCodec.encode(col.measureValues, col.measureBuf); hColValues[ci] = col.measureBuf; } return hColValues; }
public void aggregate(MeasureAggregator[] measureAggrs, List<Cell> rowCells) { int i = 0; for (int ci = 0; ci < nHCols; ci++) { HCol col = hcols[ci]; Cell cell = findCell(col, rowCells); if (cell == null) { i += col.nMeasures; continue; } ByteBuffer input = ByteBuffer.wrap(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); col.measureCodec.decode(input, col.measureValues); for (int j = 0; j < col.nMeasures; j++) measureAggrs[i++].aggregate(col.measureValues[j]); } }
public MeasureAggregator[] createBuffer() { MeasureAggregator[] aggrs = new MeasureAggregator[nTotalMeasures]; int i = 0; for (HCol col : hcols) { for (int j = 0; j < col.nMeasures; j++) aggrs[i++] = MeasureAggregator.create(col.funcNames[j], col.dataTypes[j]); } return aggrs; }
/** * @param aggrs * @param buffer byte buffer to get the metric data * @return length of metric data */ public int serializeMetricValues(MeasureAggregator[] aggrs, byte[] buffer) { for (int i = 0; i < funcNames.length; i++) { metricValues[i] = aggrs[i].getState(); } int metricBytesOffset = 0; for (int i = 0; i < measureSerializers.length; i++) { measureSerializers[i].write(metricValues[i], buffer, metricBytesOffset); metricBytesOffset += measureSerializers[i].getLength(); } return metricBytesOffset; }
@Test @SuppressWarnings("rawtypes") public void basicTest() { for (int i = 0; i < tableData.size(); ++i) { byte[] data = tableData.get(i).getBytes(); CoprocessorProjector.AggrKey aggKey = projector.getAggrKey(data); MeasureAggregator[] bufs = aggCache.getBuffer(aggKey); aggregators.aggregate(bufs, data); aggCache.checkMemoryUsage(); } assertEquals(aggCache.getAllEntries().size(), 2); long sumTotal = 0; long minTotal = 0; for (Map.Entry<CoprocessorProjector.AggrKey, MeasureAggregator[]> entry : aggCache.getAllEntries()) { sumTotal += ((LongWritable) entry.getValue()[0].getState()).get(); minTotal += ((LongWritable) entry.getValue()[1].getState()).get(); } assertEquals(3020800, sumTotal); assertEquals(1020900, minTotal); }