@Override public byte[] encode(Map<TblColRef, String> valueMap) { List<TblColRef> columns = cuboid.getColumns(); String[] values = new String[columns.size()]; for (int i = 0; i < columns.size(); i++) { values[i] = valueMap.get(columns.get(i)); } return encode(values); }
public static AbstractRowKeyEncoder createInstance(CubeSegment cubeSeg, Cuboid cuboid) { return new RowKeyEncoder(cubeSeg, cuboid); }
@Override public byte[] encode(String[] values) { byte[] bytes = new byte[this.getBytesLength()]; int offset = getHeaderLength(); for (int i = 0; i < cuboid.getColumns().size(); i++) { TblColRef column = cuboid.getColumns().get(i); int colLength = colIO.getColumnLength(column); fillColumnValue(column, colLength, values[i], bytes, offset); offset += colLength; } //fill shard and cuboid fillHeader(bytes); return bytes; }
@Override public void encode(GTRecord record, ImmutableBitSet keyColumns, byte[] buf) { ByteArray byteArray = new ByteArray(buf, getHeaderLength(), 0); encodeDims(record, keyColumns, byteArray, defaultValue()); //fill shard and cuboid fillHeader(buf); }
private void printKVInfo(PrintWriter writer) { Cuboid cuboid = Cuboid.getBaseCuboid(seg.getCubeDesc()); RowKeyEncoder encoder = new RowKeyEncoder(seg, cuboid); for (TblColRef col : cuboid.getColumns()) { writer.println("Length of dimension " + col + " is " + encoder.getColumnLength(col)); } }
@Test public void testEncodeWithoutSlr() throws Exception { CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY"); // CubeSegment seg = cube.getTheOnlySegment(); CubeDesc cubeDesc = cube.getDescriptor(); // String data = // "2013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular"; String[] data = new String[8]; data[0] = "2012-12-15"; data[1] = "11848"; data[2] = "Health & Beauty"; data[3] = "Fragrances"; data[4] = "Women"; data[5] = "FP-GTC"; data[6] = "0"; data[7] = "15"; long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId); RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid); byte[] encodedKey = rowKeyEncoder.encode(data); assertEquals(22 + rowKeyEncoder.getHeaderLength(), encodedKey.length); byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength()); byte[] rest = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), encodedKey.length); assertEquals(255, Bytes.toLong(cuboidId)); assertArrayEquals(new byte[] { 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest); }
private int buildKey(Cuboid cuboid, ByteArray[] splitBuffers) { RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(cuboid); int startIdx = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboidId int endIdx = startIdx + Long.bitCount(cuboid.getId()); int offset = 0; for (int i = startIdx; i < endIdx; i++) { System.arraycopy(splitBuffers[i].array(), splitBuffers[i].offset(), newKeyBodyBuf, offset, splitBuffers[i].length()); offset += splitBuffers[i].length(); } int fullKeySize = rowkeyEncoder.getBytesLength(); while (newKeyBuf.array().length < fullKeySize) { newKeyBuf = new ByteArray(newKeyBuf.length() * 2); } newKeyBuf.setLength(fullKeySize); rowkeyEncoder.encode(new ByteArray(newKeyBodyBuf, 0, offset), newKeyBuf); return fullKeySize; }
/** * translate GTRecord format fuzzy keys to hbase expected format * @return */ private List<Pair<byte[], byte[]>> translateFuzzyKeys(List<GTRecord> fuzzyKeys) { if (fuzzyKeys == null || fuzzyKeys.isEmpty()) { return Collections.emptyList(); } List<Pair<byte[], byte[]>> ret = Lists.newArrayList(); for (GTRecord gtRecordFuzzyKey : fuzzyKeys) { byte[] hbaseFuzzyKey = fuzzyKeyEncoder.createBuf(); byte[] hbaseFuzzyMask = fuzzyMaskEncoder.createBuf(); fuzzyKeyEncoder.encode(gtRecordFuzzyKey, gtRecordFuzzyKey.getInfo().getPrimaryKey(), hbaseFuzzyKey); fuzzyMaskEncoder.encode(gtRecordFuzzyKey, gtRecordFuzzyKey.getInfo().getPrimaryKey(), hbaseFuzzyMask); ret.add(Pair.newPair(hbaseFuzzyKey, hbaseFuzzyMask)); } return ret; }
@Override public byte[] encode(Map<TblColRef, String> valueMap) { List<byte[]> valueList = new ArrayList<byte[]>(); for (TblColRef bdCol : cuboid.getColumns()) { String value = valueMap.get(bdCol); valueList.add(valueStringToBytes(value)); } byte[][] values = valueList.toArray(RowConstants.BYTE_ARR_MARKER); return encode(values); }
@Override public void encode(ByteArray bodyBytes, ByteArray outputBuf) { Preconditions.checkState(bodyBytes.length() == bodyLength); Preconditions.checkState(bodyBytes.length() + getHeaderLength() == outputBuf.length(), // "bodybytes length: " + bodyBytes.length() + " outputBuf length: " + outputBuf.length() + " header length: " + getHeaderLength()); System.arraycopy(bodyBytes.array(), bodyBytes.offset(), outputBuf.array(), getHeaderLength(), bodyLength); //fill shard and cuboid fillHeader(outputBuf.array()); }
@Override public byte[] encode(byte[][] values) { byte[] bytes = new byte[this.bytesLength]; int offset = fillHeader(bytes, values); for (int i = 0; i < cuboid.getColumns().size(); i++) { TblColRef column = cuboid.getColumns().get(i); int colLength = colIO.getColumnLength(column); byte[] value = values[i]; if (value == null) { fillColumnValue(column, colLength, null, 0, bytes, offset); } else { fillColumnValue(column, colLength, value, value.length, bytes, offset); } offset += colLength; } return bytes; }
@Override public byte[] createBuf() { return new byte[this.getBytesLength()]; }
private void buildKeyInternal(Cuboid parentCuboid, Cuboid childCuboid, ByteArray[] splitBuffers, ByteArray newKeyBodyBuf) { RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(childCuboid); // rowkey columns long mask = Long.highestOneBit(parentCuboid.getId()); long parentCuboidId = parentCuboid.getId(); long childCuboidId = childCuboid.getId(); long parentCuboidIdActualLength = (long)Long.SIZE - Long.numberOfLeadingZeros(parentCuboid.getId()); int index = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboidId int offset = RowConstants.ROWKEY_SHARDID_LEN + RowConstants.ROWKEY_CUBOIDID_LEN; // skip shard and cuboidId for (int i = 0; i < parentCuboidIdActualLength; i++) { if ((mask & parentCuboidId) > 0) {// if the this bit position equals // 1 if ((mask & childCuboidId) > 0) {// if the child cuboid has this // column System.arraycopy(splitBuffers[index].array(), splitBuffers[index].offset(), newKeyBodyBuf.array(), offset, splitBuffers[index].length()); offset += splitBuffers[index].length(); } index++; } mask = mask >> 1; } rowkeyEncoder.fillHeader(newKeyBodyBuf.array()); }
protected void fillColumnValue(TblColRef column, int columnLen, String valueStr, byte[] outputValue, int outputValueOffset) { // special null value case if (valueStr == null) { Arrays.fill(outputValue, outputValueOffset, outputValueOffset + columnLen, defaultValue()); return; } colIO.writeColumn(column, valueStr, 0, this.blankByte, outputValue, outputValueOffset); }
public int getBytesLength() { return getHeaderLength() + bodyLength; }
RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid); byte[] encodedKey = rowKeyEncoder.encode(data); assertEquals(43 + rowKeyEncoder.getHeaderLength(), encodedKey.length); byte[] shard = Arrays.copyOfRange(encodedKey, 0, RowConstants.ROWKEY_SHARDID_LEN); @SuppressWarnings("unused") byte[] sellerId = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), 4 + rowKeyEncoder.getHeaderLength()); byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength()); byte[] rest = Arrays.copyOfRange(encodedKey, 4 + rowKeyEncoder.getHeaderLength(), encodedKey.length); assertEquals(0, Bytes.toShort(shard));
int fullKeySize = rowkeyEncoder.getBytesLength(); while (newKeyBuf.array().length < fullKeySize) { newKeyBuf = new ByteArray(newKeyBuf.length() * 2); rowkeyEncoder.encode(new ByteArray(newKeyBodyBuf, 0, bufOffset), newKeyBuf);
@Override public void encode(GTRecord record, ImmutableBitSet keyColumns, byte[] buf) { ByteArray byteArray = new ByteArray(buf, getHeaderLength(), 0); encodeDims(record, keyColumns, byteArray, defaultValue()); //fill shard and cuboid fillHeader(buf); }
/** * translate GTRecord format fuzzy keys to hbase expected format * @return */ private List<Pair<byte[], byte[]>> translateFuzzyKeys(List<GTRecord> fuzzyKeys) { if (fuzzyKeys == null || fuzzyKeys.isEmpty()) { return Collections.emptyList(); } List<Pair<byte[], byte[]>> ret = Lists.newArrayList(); for (GTRecord gtRecordFuzzyKey : fuzzyKeys) { byte[] hbaseFuzzyKey = fuzzyKeyEncoder.createBuf(); byte[] hbaseFuzzyMask = fuzzyMaskEncoder.createBuf(); fuzzyKeyEncoder.encode(gtRecordFuzzyKey, gtRecordFuzzyKey.getInfo().getPrimaryKey(), hbaseFuzzyKey); fuzzyMaskEncoder.encode(gtRecordFuzzyKey, gtRecordFuzzyKey.getInfo().getPrimaryKey(), hbaseFuzzyMask); ret.add(Pair.newPair(hbaseFuzzyKey, hbaseFuzzyMask)); } return ret; }
@Override public byte[] encode(Map<TblColRef, String> valueMap) { List<byte[]> valueList = new ArrayList<byte[]>(); for (TblColRef bdCol : cuboid.getColumns()) { String value = valueMap.get(bdCol); valueList.add(valueStringToBytes(value)); } byte[][] values = valueList.toArray(RowConstants.BYTE_ARR_MARKER); return encode(values); }