How to use

MeasureCodec

in

org.apache.kylin.metadata.measure

@Override
public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
  aggs.reset();
  for (Text value : values) {
    codec.decode(value, input);
    aggs.aggregate(input);
  }
  aggs.collectStates(result);
  valueBuf.clear();
  codec.encode(result, valueBuf);
  outputValue.set(valueBuf.array(), 0, valueBuf.position());
  context.write(key, outputValue);
  counter++;
  if (counter % BatchConstants.COUNTER_MAX == 0) {
    logger.info("Handled " + counter + " records!");
  }
}

public HCol(byte[] bFamily, byte[] bQualifier, String[] funcNames, String[] dataTypes) {
  this.family = bFamily;
  this.qualifier = bQualifier;
  this.funcNames = funcNames;
  this.dataTypes = dataTypes;
  this.nMeasures = funcNames.length;
  assert funcNames.length == dataTypes.length;
  this.measureCodec = new MeasureCodec(dataTypes);
  this.measureValues = new Object[nMeasures];
  this.measureBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
}

private void buildValue(SplittedBytes[] splitBuffers) {
  for (int i = 0; i < measures.length; i++) {
    byte[] valueBytes = getValueBytes(splitBuffers, i);
    measures[i] = measureCodec.getSerializer(i).valueOf(valueBytes);
  }
  valueBuf.clear();
  measureCodec.encode(measures, valueBuf);
}

@Test
public void basicTest() {
  MeasureDesc descs[] = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("HLLC16") };
  MeasureCodec codec = new MeasureCodec(descs);
  DoubleWritable d = new DoubleWritable(1.0);
  LongWritable l = new LongWritable(2);
  BigDecimal b = new BigDecimal("333.1234567");
  HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(16);
  hllc.add("1234567");
  hllc.add("abcdefg");
  HyperLogLogPlusCounter hllc2 = new HyperLogLogPlusCounter(16);
  hllc.add("1234567");
  hllc.add("abcdefg");
  Object values[] = new Object[] { d, l, b, hllc, hllc2 };
  ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
  codec.encode(values, buf);
  buf.flip();
  System.out.println("size: " + buf.limit());
  Object copy[] = new Object[values.length];
  codec.decode(buf, copy);
  assertTrue(Arrays.equals(values, copy));
}

public void decode(Text bytes, Object[] result) {
  decode(ByteBuffer.wrap(bytes.getBytes(), 0, bytes.getLength()), result);
}

private void verifyMeasures(List<MeasureDesc> measures, Text valueBytes, String m1, String m2, String m3) {
  MeasureCodec codec = new MeasureCodec(measures);
  Object[] values = new Object[measures.size()];
  codec.decode(valueBytes, values);
  assertTrue(new BigDecimal(m1).equals(values[0]));
  assertTrue(new BigDecimal(m2).equals(values[1]));
  assertTrue(new BigDecimal(m3).equals(values[2]));
}

@Test
public void testDecode() throws Exception {
  CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
  HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0];
  MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures());
  BigDecimal sum = new BigDecimal("333.1234567");
  BigDecimal min = new BigDecimal("333.1111111");
  BigDecimal max = new BigDecimal("333.1999999");
  LongWritable count = new LongWritable(2);
  ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
  codec.encode(new Object[] { sum, min, max, count }, buf);
  buf.flip();
  byte[] valueBytes = new byte[buf.limit()];
  System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());
  RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
  for (MeasureDesc measure : cubeDesc.getMeasures()) {
    FunctionDesc aggrFunc = measure.getFunction();
    int index = hbaseCol.findMeasureIndex(aggrFunc);
    rowValueDecoder.setIndex(index);
  }
  rowValueDecoder.decode(valueBytes);
  List<String> measureNames = rowValueDecoder.getNames();
  Object[] measureValues = rowValueDecoder.getValues();
  assertEquals("[PRICE, MIN_PRICE_, MAX_PRICE_, COUNT__]", measureNames.toString());
  assertEquals("[333.1234567, 333.1111111, 333.1999999, 2]", Arrays.toString(measureValues));
}

public ByteBuffer[] getHColValues(MeasureAggregator[] aggrs) {
  int i = 0;
  for (int ci = 0; ci < nHCols; ci++) {
    HCol col = hcols[ci];
    for (int j = 0; j < col.nMeasures; j++)
      col.measureValues[j] = aggrs[i++].getState();
    col.measureBuf.clear();
    col.measureCodec.encode(col.measureValues, col.measureBuf);
    hColValues[ci] = col.measureBuf;
  }
  return hColValues;
}

public MeasureCodec(String... dataTypes) {
  init(dataTypes);
}

public void decode(byte[] bytes) {
  codec.decode(ByteBuffer.wrap(bytes), values);
  convertToJavaObjects(values, values);
}

public KeyValue create(Text key, Object[] measureValues) {
  for (int i = 0; i < colValues.length; i++) {
    colValues[i] = measureValues[refIndex[i]];
  }
  valueBuf.clear();
  codec.encode(colValues, valueBuf);
  return create(key, valueBuf.array(), 0, valueBuf.position());
}

public MeasureCodec(MeasureDesc... measureDescs) {
  String[] dataTypes = new String[measureDescs.length];
  for (int i = 0; i < dataTypes.length; i++) {
    dataTypes[i] = measureDescs[i].getFunction().getReturnType();
  }
  init(dataTypes);
}

public RowValueDecoder(HBaseColumnDesc hbaseColumn) {
  this.hbaseColumn = hbaseColumn;
  this.projectionIndex = new BitSet();
  this.names = new ArrayList<String>();
  this.measures = hbaseColumn.getMeasures();
  for (MeasureDesc measure : measures) {
    this.names.add(measure.getFunction().getRewriteFieldName());
  }
  this.codec = new MeasureCodec(measures);
  this.values = new Object[measures.length];
}

@Override
public void map(Text key, Text value, Context context) throws IOException, InterruptedException {
  outputKey.set(key.getBytes(), 0, key.getLength());
  KeyValue outputValue;
  int n = keyValueCreators.size();
  if (n == 1 && keyValueCreators.get(0).isFullCopy) { // shortcut for
                            // simple full copy
    outputValue = keyValueCreators.get(0).create(key, value.getBytes(), 0, value.getLength());
    context.write(outputKey, outputValue);
  } else { // normal (complex) case that distributes measures to multiple
       // HBase columns
    inputCodec.decode(value, inputMeasures);
    for (int i = 0; i < n; i++) {
      outputValue = keyValueCreators.get(i).create(key, inputMeasures);
      context.write(outputKey, outputValue);
    }
  }
}

private Text newValueText(MeasureCodec codec, String sum, String min, String max, int count) {
  Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new LongWritable(count) };
  buf.clear();
  codec.encode(values, buf);
  Text t = new Text();
  t.set(buf.array(), 0, buf.position());
  return t;
}

private void buildValue(SplittedBytes[] splitBuffers) {
  for (int i = 0; i < measures.length; i++) {
    byte[] valueBytes = getValueBytes(splitBuffers, i);
    measures[i] = measureCodec.getSerializer(i).valueOf(valueBytes);
  }
  valueBuf.clear();
  measureCodec.encode(measures, valueBuf);
}

public RowValueDecoder(RowValueDecoder rowValueDecoder) {
  this.hbaseColumn = rowValueDecoder.getHBaseColumn();
  this.projectionIndex = rowValueDecoder.getProjectionIndex();
  this.names = new ArrayList<String>();
  this.measures = hbaseColumn.getMeasures();
  for (MeasureDesc measure : measures) {
    this.names.add(measure.getFunction().getRewriteFieldName());
  }
  this.codec = new MeasureCodec(measures);
  this.values = new Object[measures.length];
}

public void aggregate(MeasureAggregator[] measureAggrs, List<Cell> rowCells) {
  int i = 0;
  for (int ci = 0; ci < nHCols; ci++) {
    HCol col = hcols[ci];
    Cell cell = findCell(col, rowCells);
    if (cell == null) {
      i += col.nMeasures;
      continue;
    }
    ByteBuffer input = ByteBuffer.wrap(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
    col.measureCodec.decode(input, col.measureValues);
    for (int j = 0; j < col.nMeasures; j++)
      measureAggrs[i++].aggregate(col.measureValues[j]);
  }
}

private Cell newCell(byte[] key, HCol col, String decimal, int number) {
  Object[] values = number == Integer.MIN_VALUE ? //
  new Object[] { new BigDecimal(decimal) } //
      : new Object[] { new BigDecimal(decimal), new LongWritable(number) };
  buf.clear();
  col.measureCodec.encode(values, buf);
  Cell keyValue = new KeyValue(key, 0, key.length, //
      col.family, 0, col.family.length, //
      col.qualifier, 0, col.qualifier.length, //
      HConstants.LATEST_TIMESTAMP, Type.Put, //
      buf.array(), 0, buf.position());
  return keyValue;
}

public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) {
  cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
  qBytes = Bytes.toBytes(colDesc.getQualifier());
  timestamp = System.currentTimeMillis();
  List<MeasureDesc> measures = cubeDesc.getMeasures();
  String[] measureNames = getMeasureNames(cubeDesc);
  String[] refs = colDesc.getMeasureRefs();
  refIndex = new int[refs.length];
  refMeasures = new MeasureDesc[refs.length];
  for (int i = 0; i < refs.length; i++) {
    refIndex[i] = indexOf(measureNames, refs[i]);
    refMeasures[i] = measures.get(refIndex[i]);
  }
  codec = new MeasureCodec(refMeasures);
  colValues = new Object[refs.length];
  isFullCopy = true;
  for (int i = 0; i < measures.size(); i++) {
    if (refIndex.length <= i || refIndex[i] != i)
      isFullCopy = false;
  }
}