public HCol(byte[] bFamily, byte[] bQualifier, String[] funcNames, String[] dataTypes) { this.family = bFamily; this.qualifier = bQualifier; this.funcNames = funcNames; this.dataTypes = dataTypes; this.nMeasures = funcNames.length; assert funcNames.length == dataTypes.length; this.measureCodec = new MeasureCodec(dataTypes); this.measureValues = new Object[nMeasures]; this.measureBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); }
public RowValueDecoder(HBaseColumnDesc hbaseColumn) { this.hbaseColumn = hbaseColumn; this.projectionIndex = new BitSet(); this.names = new ArrayList<String>(); this.measures = hbaseColumn.getMeasures(); for (MeasureDesc measure : measures) { this.names.add(measure.getFunction().getRewriteFieldName()); } this.codec = new MeasureCodec(measures); this.values = new Object[measures.length]; }
public RowValueDecoder(RowValueDecoder rowValueDecoder) { this.hbaseColumn = rowValueDecoder.getHBaseColumn(); this.projectionIndex = rowValueDecoder.getProjectionIndex(); this.names = new ArrayList<String>(); this.measures = hbaseColumn.getMeasures(); for (MeasureDesc measure : measures) { this.names.add(measure.getFunction().getRewriteFieldName()); } this.codec = new MeasureCodec(measures); this.values = new Object[measures.length]; }
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) { cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName()); qBytes = Bytes.toBytes(colDesc.getQualifier()); timestamp = System.currentTimeMillis(); List<MeasureDesc> measures = cubeDesc.getMeasures(); String[] measureNames = getMeasureNames(cubeDesc); String[] refs = colDesc.getMeasureRefs(); refIndex = new int[refs.length]; refMeasures = new MeasureDesc[refs.length]; for (int i = 0; i < refs.length; i++) { refIndex[i] = indexOf(measureNames, refs[i]); refMeasures[i] = measures.get(refIndex[i]); } codec = new MeasureCodec(refMeasures); colValues = new Object[refs.length]; isFullCopy = true; for (int i = 0; i < measures.size(); i++) { if (refIndex.length <= i || refIndex[i] != i) isFullCopy = false; } }
@Override protected void setup(Context context) throws IOException { super.publishConfiguration(context.getConfiguration()); cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase(); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration()); cubeDesc = CubeManager.getInstance(config).getCube(cubeName).getDescriptor(); measuresDescs = cubeDesc.getMeasures(); codec = new MeasureCodec(measuresDescs); aggs = new MeasureAggregators(measuresDescs); input = new Object[measuresDescs.size()]; result = new Object[measuresDescs.size()]; }
@Override protected void setup(Context context) throws IOException { super.publishConfiguration(context.getConfiguration()); cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration()); CubeManager cubeMgr = CubeManager.getInstance(config); cubeDesc = cubeMgr.getCube(cubeName).getDescriptor(); inputCodec = new MeasureCodec(cubeDesc.getMeasures()); inputMeasures = new Object[cubeDesc.getMeasures().size()]; keyValueCreators = Lists.newArrayList(); for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) { for (HBaseColumnDesc colDesc : cfDesc.getColumns()) { keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc)); } } }
@Override protected void setup(Context context) throws IOException { super.publishConfiguration(context.getConfiguration()); cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase(); segmentName = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_NAME); intermediateTableRowDelimiter = context.getConfiguration().get(BatchConstants.CFG_CUBE_INTERMEDIATE_TABLE_ROW_DELIMITER, Character.toString(BatchConstants.INTERMEDIATE_TABLE_ROW_DELIMITER)); if (Bytes.toBytes(intermediateTableRowDelimiter).length > 1) { throw new RuntimeException("Expected delimiter byte length is 1, but got " + Bytes.toBytes(intermediateTableRowDelimiter).length); } byteRowDelimiter = Bytes.toBytes(intermediateTableRowDelimiter)[0]; KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration()); cube = CubeManager.getInstance(config).getCube(cubeName); cubeDesc = cube.getDescriptor(); cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.NEW); long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId); intermediateTableDesc = new CubeJoinedFlatTableDesc(cube.getDescriptor(), cubeSegment); bytesSplitter = new BytesSplitter(200, 4096); rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid); measureCodec = new MeasureCodec(cubeDesc.getMeasures()); measures = new Object[cubeDesc.getMeasures().size()]; int colCount = cubeDesc.getRowkey().getRowKeyColumns().length; keyBytesBuf = new byte[colCount][]; initNullBytes(); }
private void verifyMeasures(List<MeasureDesc> measures, Text valueBytes, String m1, String m2, String m3) { MeasureCodec codec = new MeasureCodec(measures); Object[] values = new Object[measures.size()]; codec.decode(valueBytes, values); assertTrue(new BigDecimal(m1).equals(values[0])); assertTrue(new BigDecimal(m2).equals(values[1])); assertTrue(new BigDecimal(m3).equals(values[2])); }
@Override protected void setup(Context context) throws IOException { super.publishConfiguration(context.getConfiguration()); cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase(); segmentName = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_NAME); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration()); metadataManager = MetadataManager.getInstance(config); cube = CubeManager.getInstance(config).getCube(cubeName); cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.NEW); cubeDesc = cube.getDescriptor(); factTableDesc = metadataManager.getTableDesc(cubeDesc.getFactTable()); long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId); // intermediateTableDesc = new // JoinedFlatTableDesc(cube.getDescriptor()); rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid); measureCodec = new MeasureCodec(cubeDesc.getMeasures()); measures = new Object[cubeDesc.getMeasures().size()]; int colCount = cubeDesc.getRowkey().getRowKeyColumns().length; keyBytesBuf = new byte[colCount][]; bytesSplitter = new BytesSplitter(factTableDesc.getColumns().length, 4096); nullValue = new byte[] { (byte) '\\', (byte) 'N' };// As in Hive, null // value is // represented by \N prepareJoins(); prepareMetrics(); }
@Before public void setup() throws Exception { this.createTestMetadata(); // hack for distributed cache FileUtils.deleteDirectory(new File("../job/meta")); FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta")); CubeDesc desc = CubeManager.getInstance(getTestConfig()).getCube(cubeName).getDescriptor(); codec = new MeasureCodec(desc.getMeasures()); }
@Test public void testReducer() throws Exception { reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready"); CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); MeasureCodec codec = new MeasureCodec(cubeDesc.getMeasures()); Text key1 = new Text("72010ustech"); List<Text> values1 = new ArrayList<Text>(); values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1)); values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1)); values1.add(newValueText(codec, "10", "10", "10", 1)); Text key2 = new Text("1tech"); List<Text> values2 = new ArrayList<Text>(); values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1)); values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1)); Text key3 = new Text("0"); List<Text> values3 = new ArrayList<Text>(); values3.add(newValueText(codec, "146.52", "146.52", "146.52", 4)); reduceDriver.withInput(key1, values1); reduceDriver.withInput(key2, values2); reduceDriver.withInput(key3, values3); List<Pair<Text, Text>> result = reduceDriver.run(); Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "45.43", "10", "20.34", 3)); Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "35.43", "15.09", "20.34", 2)); Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "146.52", "146.52", "146.52", 4)); assertEquals(3, result.size()); assertTrue(result.contains(p1)); assertTrue(result.contains(p2)); assertTrue(result.contains(p3)); }
@Test public void testDecode() throws Exception { CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0]; MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures()); BigDecimal sum = new BigDecimal("333.1234567"); BigDecimal min = new BigDecimal("333.1111111"); BigDecimal max = new BigDecimal("333.1999999"); LongWritable count = new LongWritable(2); ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); codec.encode(new Object[] { sum, min, max, count }, buf); buf.flip(); byte[] valueBytes = new byte[buf.limit()]; System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit()); RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol); for (MeasureDesc measure : cubeDesc.getMeasures()) { FunctionDesc aggrFunc = measure.getFunction(); int index = hbaseCol.findMeasureIndex(aggrFunc); rowValueDecoder.setIndex(index); } rowValueDecoder.decode(valueBytes); List<String> measureNames = rowValueDecoder.getNames(); Object[] measureValues = rowValueDecoder.getValues(); assertEquals("[PRICE, MIN_PRICE_, MAX_PRICE_, COUNT__]", measureNames.toString()); assertEquals("[333.1234567, 333.1111111, 333.1999999, 2]", Arrays.toString(measureValues)); }
@Test public void basicTest() { MeasureDesc descs[] = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("HLLC16") }; MeasureCodec codec = new MeasureCodec(descs); DoubleWritable d = new DoubleWritable(1.0); LongWritable l = new LongWritable(2); BigDecimal b = new BigDecimal("333.1234567"); HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(16); hllc.add("1234567"); hllc.add("abcdefg"); HyperLogLogPlusCounter hllc2 = new HyperLogLogPlusCounter(16); hllc.add("1234567"); hllc.add("abcdefg"); Object values[] = new Object[] { d, l, b, hllc, hllc2 }; ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); codec.encode(values, buf); buf.flip(); System.out.println("size: " + buf.limit()); Object copy[] = new Object[values.length]; codec.decode(buf, copy); assertTrue(Arrays.equals(values, copy)); }