@Test public void testDecodeWithoutSlr() throws Exception { CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY"); RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment()); byte[] key = { 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }; rowKeyDecoder.decode(key); List<String> values = rowKeyDecoder.getValues(); assertEquals("[" + millis("2012-12-15") + ", 11848, Health & Beauty, Fragrances, Women, FP-GTC, 0, 15]", values.toString()); }
public long decode(byte[] bytes) throws IOException { this.values.clear(); long cuboidId = rowKeySplitter.split(bytes); initCuboid(cuboidId); ByteArray[] splits = rowKeySplitter.getSplitBuffers(); int offset = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboid id part for (int i = 0; i < this.cuboid.getColumns().size(); i++) { TblColRef col = this.cuboid.getColumns().get(i); collectValue(col, splits[offset].array(), splits[offset].offset(), splits[offset].length()); offset++; } return cuboidId; }
@Override public void doMap(Text key, Text value, Context context) throws InterruptedException, IOException { long cuboidID = rowKeyDecoder.decode(key.getBytes()); if (cuboidID != baseCuboidId) { return; // Skip data from cuboids which are not the base cuboid } List<String> keyValues = rowKeyDecoder.getValues(); if (rowCount < samplingPercentage) { Preconditions.checkArgument(nRowKey == keyValues.size()); String[] row = keyValues.toArray(new String[keyValues.size()]); if (isUsePutRowKeyToHllNewAlgorithm) { putRowKeyToHLLNew(row); } else { putRowKeyToHLLOld(row); } } if (++rowCount == 100) rowCount = 0; }
@Test public void testDecodeWithoutSlr() throws Exception { CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY"); RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment()); byte[] key = { 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }; rowKeyDecoder.decode(key); List<String> names = rowKeyDecoder.getNames(null); List<String> values = rowKeyDecoder.getValues(); assertEquals("[CAL_DT, LEAF_CATEG_ID, META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME, LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD]", names.toString()); assertEquals("[2012-12-15, 11848, Health & Beauty, Fragrances, Women, FP-GTC, 0, 15]", values.toString()); }
private void translateResult(Result res, Tuple tuple) throws IOException { rowKeyDecoder.decode(rowkey); List<TblColRef> columns = rowKeyDecoder.getColumns(); List<String> dimensionNames = rowKeyDecoder.getNames(context.getAliasMap()); List<String> dimensionValues = rowKeyDecoder.getValues(); for (int i = 0; i < dimensionNames.size(); i++) { TblColRef column = columns.get(i);
rowKeyDecoder = new RowKeyDecoder(cubeSegment);
private TupleInfo buildTupleInfo(Cuboid cuboid) { TupleInfo info = new TupleInfo(); int index = 0; rowKeyDecoder.setCuboid(cuboid); List<TblColRef> rowColumns = rowKeyDecoder.getColumns(); List<String> colNames = rowKeyDecoder.getNames(context.getAliasMap()); for (int i = 0; i < rowColumns.size(); i++) { TblColRef column = rowColumns.get(i);
private List<String> buildNameList(Map<TblColRef, String> aliasMap) { List<TblColRef> columnList = getColumns(); List<String> result = new ArrayList<String>(columnList.size()); for (TblColRef col : columnList) result.add(findName(col, aliasMap)); return result; }
@Test public void testDecodeWithSlr() throws Exception { CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY"); RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment()); byte[] key = { 0, 0, 0, 0, 0, 0, 1, -1, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 54, -105, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 }; rowKeyDecoder.decode(key); List<String> names = rowKeyDecoder.getNames(null); List<String> values = rowKeyDecoder.getValues(); assertEquals("[SELLER_ID, CAL_DT, LEAF_CATEG_ID, META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME, LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD]", names.toString()); assertEquals("[10000000, 2012-01-02, 20213, Collectibles, Postcards, US StateCities & Towns, ABIN, 0, -99]", values.toString()); }
@Override public void doMap(Text key, Text value, Context context) throws InterruptedException, IOException { long cuboidID = rowKeyDecoder.decode(key.getBytes()); if (cuboidID != baseCuboidId) { return; // Skip data from cuboids which are not the base cuboid } List<String> keyValues = rowKeyDecoder.getValues(); if (rowCount < samplingPercentage) { Preconditions.checkArgument(nRowKey == keyValues.size()); String[] row = keyValues.toArray(new String[keyValues.size()]); if (isUsePutRowKeyToHllNewAlgorithm) { putRowKeyToHLLNew(row); } else { putRowKeyToHLLOld(row); } } if (++rowCount == 100) rowCount = 0; }
public CubeSegmentTupleIterator(CubeSegment cubeSeg, Collection<HBaseKeyRange> keyRanges, HConnection conn, Collection<TblColRef> dimensions, TupleFilter filter, Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context) { this.cube = cubeSeg.getCubeInstance(); this.cubeSeg = cubeSeg; this.dimensions = dimensions; this.filter = filter; this.groupBy = groupBy; this.rowValueDecoders = rowValueDecoders; this.context = context; this.tableName = cubeSeg.getStorageLocationIdentifier(); this.rowKeyDecoder = new RowKeyDecoder(this.cubeSeg); this.scanCount = 0; try { this.table = conn.getTable(tableName); } catch (Throwable t) { throw new StorageException("Error when open connection to table " + tableName, t); } this.rangeIterator = keyRanges.iterator(); scanNextRange(); }
private List<String> buildNameList(Map<TblColRef, String> aliasMap) { List<TblColRef> columnList = getColumns(); List<String> result = new ArrayList<String>(columnList.size()); for (TblColRef col : columnList) result.add(findName(col, aliasMap)); return result; }
@Ignore @Test public void testDecodeWithSlr() throws Exception { CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY"); RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment()); byte[] key = { 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, -104, -106, -128, 11, 54, -105, 50, 48, 50, 49, 51, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 }; rowKeyDecoder.decode(key); List<String> values = rowKeyDecoder.getValues(); assertEquals("[10000000, " + millis("2012-01-02") + ", 20213, Collectibles, Postcards, US StateCities & Towns, ABIN, 0, -99]", values.toString()); }
@Test public void testEncodeAndDecodeWithUtf8() throws IOException { CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY"); CubeDesc cubeDesc = cube.getDescriptor(); byte[][] data = new byte[8][]; data[0] = Bytes.toBytes("2012-12-15"); data[1] = Bytes.toBytes("11848"); data[2] = Bytes.toBytes("Health & Beauty"); data[3] = Bytes.toBytes("Fragrances"); data[4] = Bytes.toBytes("Women"); data[5] = Bytes.toBytes("刊登格式测试");// UTF-8 data[6] = Bytes.toBytes("0"); data[7] = Bytes.toBytes("15"); long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId); AbstractRowKeyEncoder rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cube.getFirstSegment(), baseCuboid); byte[] encodedKey = rowKeyEncoder.encode(data); assertEquals(30, encodedKey.length); RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment()); rowKeyDecoder.decode(encodedKey); List<String> names = rowKeyDecoder.getNames(null); List<String> values = rowKeyDecoder.getValues(); assertEquals("[CAL_DT, LEAF_CATEG_ID, META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME, LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD]", names.toString()); assertEquals("[2012-12-15, 11848, Health & Beauty, Fragrances, Women, 刊登格式, 0, 15]", values.toString()); } }
rowKeyDecoder = new RowKeyDecoder(cubeSegment);
public long decode(byte[] bytes) throws IOException { this.values.clear(); long cuboidId = rowKeySplitter.split(bytes, bytes.length); initCuboid(cuboidId); SplittedBytes[] splits = rowKeySplitter.getSplitBuffers(); int offset = 1; // skip cuboid id part for (int i = 0; i < this.cuboid.getColumns().size(); i++) { TblColRef col = this.cuboid.getColumns().get(i); collectValue(col, splits[offset].value, splits[offset].length); offset++; } return cuboidId; }
@Test public void testEncodeAndDecodeWithUtf8() throws IOException { CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY"); CubeDesc cubeDesc = cube.getDescriptor(); String[] data = new String[8]; data[0] = "2012-12-15"; data[1] = "11848"; data[2] = "Health & Beauty"; data[3] = "Fragrances"; data[4] = "Women"; data[5] = "刊登格式测试";// UTF-8 data[6] = "0"; data[7] = "15"; long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId); RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid); byte[] encodedKey = rowKeyEncoder.encode(data); assertEquals(22 + rowKeyEncoder.getHeaderLength(), encodedKey.length); RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment()); rowKeyDecoder.decode(encodedKey); List<String> values = rowKeyDecoder.getValues(); assertEquals("[" + millis("2012-12-15") + ", 11848, Health & Beauty, Fragrances, Women, 刊登格式, 0, 15]", values.toString()); }
public long decode(byte[] bytes) throws IOException { this.values.clear(); long cuboidId = rowKeySplitter.split(bytes, bytes.length); initCuboid(cuboidId); SplittedBytes[] splits = rowKeySplitter.getSplitBuffers(); int offset = 1; // skip cuboid id part for (int i = 0; i < this.cuboid.getColumns().size(); i++) { TblColRef col = this.cuboid.getColumns().get(i); collectValue(col, splits[offset].value, splits[offset].length); offset++; } return cuboidId; }
@Test public void testMapperWithHeader() throws Exception { String cubeName = "test_kylin_cube_with_slr_1_new_segment"; String segmentName = "20130331080000_20131212080000"; mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName); mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName); // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL, // metadata); mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrancesWomenAuction15123456789132.33")); List<Pair<Text, Text>> result = mapDriver.run(); CubeManager cubeMgr = CubeManager.getInstance(getTestConfig()); CubeInstance cube = cubeMgr.getCube(cubeName); assertEquals(1, result.size()); Text rowkey = result.get(0).getFirst(); byte[] key = rowkey.getBytes(); byte[] header = Bytes.head(key, 26); byte[] sellerId = Bytes.tail(header, 18); byte[] cuboidId = Bytes.head(header, 8); byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26); RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment()); decoder.decode(key); assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, Women, Auction, 0, 15]", decoder.getValues().toString()); assertTrue(Bytes.toString(sellerId).startsWith("123456789")); assertEquals(511, Bytes.toLong(cuboidId)); assertEquals(22, restKey.length); verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "132.33", "132.33", "132.33"); }
public long decode(byte[] bytes) throws IOException { this.values.clear(); long cuboidId = rowKeySplitter.split(bytes); initCuboid(cuboidId); ByteArray[] splits = rowKeySplitter.getSplitBuffers(); int offset = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboid id part for (int i = 0; i < this.cuboid.getColumns().size(); i++) { TblColRef col = this.cuboid.getColumns().get(i); collectValue(col, splits[offset].array(), splits[offset].offset(), splits[offset].length()); offset++; } return cuboidId; }