@Override public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException { aggs.reset(); for (Text value : values) { codec.decode(value, input); aggs.aggregate(input); } aggs.collectStates(result); valueBuf.clear(); codec.encode(result, valueBuf); outputValue.set(valueBuf.array(), 0, valueBuf.position()); context.write(key, outputValue); counter++; if (counter % BatchConstants.COUNTER_MAX == 0) { logger.info("Handled " + counter + " records!"); } }
public HCol(byte[] bFamily, byte[] bQualifier, String[] funcNames, String[] dataTypes) { this.family = bFamily; this.qualifier = bQualifier; this.funcNames = funcNames; this.dataTypes = dataTypes; this.nMeasures = funcNames.length; assert funcNames.length == dataTypes.length; this.measureCodec = new MeasureCodec(dataTypes); this.measureValues = new Object[nMeasures]; this.measureBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); }
private void buildValue(SplittedBytes[] splitBuffers) { for (int i = 0; i < measures.length; i++) { byte[] valueBytes = getValueBytes(splitBuffers, i); measures[i] = measureCodec.getSerializer(i).valueOf(valueBytes); } valueBuf.clear(); measureCodec.encode(measures, valueBuf); }
@Test public void basicTest() { MeasureDesc descs[] = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("HLLC16") }; MeasureCodec codec = new MeasureCodec(descs); DoubleWritable d = new DoubleWritable(1.0); LongWritable l = new LongWritable(2); BigDecimal b = new BigDecimal("333.1234567"); HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(16); hllc.add("1234567"); hllc.add("abcdefg"); HyperLogLogPlusCounter hllc2 = new HyperLogLogPlusCounter(16); hllc.add("1234567"); hllc.add("abcdefg"); Object values[] = new Object[] { d, l, b, hllc, hllc2 }; ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); codec.encode(values, buf); buf.flip(); System.out.println("size: " + buf.limit()); Object copy[] = new Object[values.length]; codec.decode(buf, copy); assertTrue(Arrays.equals(values, copy)); }
public void decode(Text bytes, Object[] result) { decode(ByteBuffer.wrap(bytes.getBytes(), 0, bytes.getLength()), result); }
private void verifyMeasures(List<MeasureDesc> measures, Text valueBytes, String m1, String m2, String m3) { MeasureCodec codec = new MeasureCodec(measures); Object[] values = new Object[measures.size()]; codec.decode(valueBytes, values); assertTrue(new BigDecimal(m1).equals(values[0])); assertTrue(new BigDecimal(m2).equals(values[1])); assertTrue(new BigDecimal(m3).equals(values[2])); }
@Test public void testDecode() throws Exception { CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0]; MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures()); BigDecimal sum = new BigDecimal("333.1234567"); BigDecimal min = new BigDecimal("333.1111111"); BigDecimal max = new BigDecimal("333.1999999"); LongWritable count = new LongWritable(2); ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); codec.encode(new Object[] { sum, min, max, count }, buf); buf.flip(); byte[] valueBytes = new byte[buf.limit()]; System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit()); RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol); for (MeasureDesc measure : cubeDesc.getMeasures()) { FunctionDesc aggrFunc = measure.getFunction(); int index = hbaseCol.findMeasureIndex(aggrFunc); rowValueDecoder.setIndex(index); } rowValueDecoder.decode(valueBytes); List<String> measureNames = rowValueDecoder.getNames(); Object[] measureValues = rowValueDecoder.getValues(); assertEquals("[PRICE, MIN_PRICE_, MAX_PRICE_, COUNT__]", measureNames.toString()); assertEquals("[333.1234567, 333.1111111, 333.1999999, 2]", Arrays.toString(measureValues)); }
public ByteBuffer[] getHColValues(MeasureAggregator[] aggrs) { int i = 0; for (int ci = 0; ci < nHCols; ci++) { HCol col = hcols[ci]; for (int j = 0; j < col.nMeasures; j++) col.measureValues[j] = aggrs[i++].getState(); col.measureBuf.clear(); col.measureCodec.encode(col.measureValues, col.measureBuf); hColValues[ci] = col.measureBuf; } return hColValues; }
public MeasureCodec(String... dataTypes) { init(dataTypes); }
public void decode(byte[] bytes) { codec.decode(ByteBuffer.wrap(bytes), values); convertToJavaObjects(values, values); }
public KeyValue create(Text key, Object[] measureValues) { for (int i = 0; i < colValues.length; i++) { colValues[i] = measureValues[refIndex[i]]; } valueBuf.clear(); codec.encode(colValues, valueBuf); return create(key, valueBuf.array(), 0, valueBuf.position()); }
public MeasureCodec(MeasureDesc... measureDescs) { String[] dataTypes = new String[measureDescs.length]; for (int i = 0; i < dataTypes.length; i++) { dataTypes[i] = measureDescs[i].getFunction().getReturnType(); } init(dataTypes); }
public RowValueDecoder(HBaseColumnDesc hbaseColumn) { this.hbaseColumn = hbaseColumn; this.projectionIndex = new BitSet(); this.names = new ArrayList<String>(); this.measures = hbaseColumn.getMeasures(); for (MeasureDesc measure : measures) { this.names.add(measure.getFunction().getRewriteFieldName()); } this.codec = new MeasureCodec(measures); this.values = new Object[measures.length]; }
@Override public void map(Text key, Text value, Context context) throws IOException, InterruptedException { outputKey.set(key.getBytes(), 0, key.getLength()); KeyValue outputValue; int n = keyValueCreators.size(); if (n == 1 && keyValueCreators.get(0).isFullCopy) { // shortcut for // simple full copy outputValue = keyValueCreators.get(0).create(key, value.getBytes(), 0, value.getLength()); context.write(outputKey, outputValue); } else { // normal (complex) case that distributes measures to multiple // HBase columns inputCodec.decode(value, inputMeasures); for (int i = 0; i < n; i++) { outputValue = keyValueCreators.get(i).create(key, inputMeasures); context.write(outputKey, outputValue); } } }
private Text newValueText(MeasureCodec codec, String sum, String min, String max, int count) { Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new LongWritable(count) }; buf.clear(); codec.encode(values, buf); Text t = new Text(); t.set(buf.array(), 0, buf.position()); return t; }
private void buildValue(SplittedBytes[] splitBuffers) { for (int i = 0; i < measures.length; i++) { byte[] valueBytes = getValueBytes(splitBuffers, i); measures[i] = measureCodec.getSerializer(i).valueOf(valueBytes); } valueBuf.clear(); measureCodec.encode(measures, valueBuf); }
public RowValueDecoder(RowValueDecoder rowValueDecoder) { this.hbaseColumn = rowValueDecoder.getHBaseColumn(); this.projectionIndex = rowValueDecoder.getProjectionIndex(); this.names = new ArrayList<String>(); this.measures = hbaseColumn.getMeasures(); for (MeasureDesc measure : measures) { this.names.add(measure.getFunction().getRewriteFieldName()); } this.codec = new MeasureCodec(measures); this.values = new Object[measures.length]; }
public void aggregate(MeasureAggregator[] measureAggrs, List<Cell> rowCells) { int i = 0; for (int ci = 0; ci < nHCols; ci++) { HCol col = hcols[ci]; Cell cell = findCell(col, rowCells); if (cell == null) { i += col.nMeasures; continue; } ByteBuffer input = ByteBuffer.wrap(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); col.measureCodec.decode(input, col.measureValues); for (int j = 0; j < col.nMeasures; j++) measureAggrs[i++].aggregate(col.measureValues[j]); } }
private Cell newCell(byte[] key, HCol col, String decimal, int number) { Object[] values = number == Integer.MIN_VALUE ? // new Object[] { new BigDecimal(decimal) } // : new Object[] { new BigDecimal(decimal), new LongWritable(number) }; buf.clear(); col.measureCodec.encode(values, buf); Cell keyValue = new KeyValue(key, 0, key.length, // col.family, 0, col.family.length, // col.qualifier, 0, col.qualifier.length, // HConstants.LATEST_TIMESTAMP, Type.Put, // buf.array(), 0, buf.position()); return keyValue; }
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) { cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName()); qBytes = Bytes.toBytes(colDesc.getQualifier()); timestamp = System.currentTimeMillis(); List<MeasureDesc> measures = cubeDesc.getMeasures(); String[] measureNames = getMeasureNames(cubeDesc); String[] refs = colDesc.getMeasureRefs(); refIndex = new int[refs.length]; refMeasures = new MeasureDesc[refs.length]; for (int i = 0; i < refs.length; i++) { refIndex[i] = indexOf(measureNames, refs[i]); refMeasures[i] = measures.get(refIndex[i]); } codec = new MeasureCodec(refMeasures); colValues = new Object[refs.length]; isFullCopy = true; for (int i = 0; i < measures.size(); i++) { if (refIndex.length <= i || refIndex[i] != i) isFullCopy = false; } }