/** * translate GTRecord format fuzzy keys to hbase expected format * @return */ private List<Pair<byte[], byte[]>> translateFuzzyKeys(List<GTRecord> fuzzyKeys) { if (fuzzyKeys == null || fuzzyKeys.isEmpty()) { return Collections.emptyList(); } List<Pair<byte[], byte[]>> ret = Lists.newArrayList(); for (GTRecord gtRecordFuzzyKey : fuzzyKeys) { byte[] hbaseFuzzyKey = fuzzyKeyEncoder.createBuf(); byte[] hbaseFuzzyMask = fuzzyMaskEncoder.createBuf(); fuzzyKeyEncoder.encode(gtRecordFuzzyKey, gtRecordFuzzyKey.getInfo().getPrimaryKey(), hbaseFuzzyKey); fuzzyMaskEncoder.encode(gtRecordFuzzyKey, gtRecordFuzzyKey.getInfo().getPrimaryKey(), hbaseFuzzyMask); ret.add(Pair.newPair(hbaseFuzzyKey, hbaseFuzzyMask)); } return ret; }
private ColumnTupleFilter col(int i) { return new ColumnTupleFilter(info.colRef(i)); }
@Override public void encode(GTRecord record, ImmutableBitSet keyColumns, byte[] buf) { ByteArray byteArray = new ByteArray(buf, getHeaderLength(), 0); GTInfo info = record.getInfo(); byte fill; int pos = 0; for (int i = 0; i < info.getPrimaryKey().trueBitCount(); i++) { int c = info.getPrimaryKey().trueBitAt(i); int colLength = info.getCodeSystem().maxCodeLength(c); if (record.get(c).array() != null) { fill = RowConstants.BYTE_ZERO; } else { fill = RowConstants.BYTE_ONE; } Arrays.fill(byteArray.array(), byteArray.offset() + pos, byteArray.offset() + pos + colLength, fill); pos += colLength; } byteArray.setLength(pos); //fill shard and cuboid fillHeader(buf); }
private boolean isMemoryHungry(ImmutableBitSet selectedColBlocks) { ImmutableBitSet selectColumns = fullGTInfo.selectColumns(selectedColBlocks); return fullGTInfo.getMaxColumnLength(selectColumns) > 1024; }
private void validate() { if (info.getColumnCount() != colSpecs.size()) throw new IllegalArgumentException(); for (int i = 0; i < colSpecs.size(); i++) { ColSpec spec = colSpecs.get(i); if (info.getCodeSystem().maxCodeLength(i) < spec.length) throw new IllegalArgumentException(); } }
/** create a row block, allocate memory, get ready for writing */ public static GTRowBlock allocate(GTInfo info) { GTRowBlock b = new GTRowBlock(info); byte[] array = new byte[info.getMaxColumnLength(info.primaryKey)]; b.primaryKey.reset(array, 0, array.length); int maxRows = info.isRowBlockEnabled() ? info.rowBlockSize : 1; for (int i = 0; i < b.cellBlocks.length; i++) { array = new byte[info.getMaxColumnLength(info.colBlocks[i]) * maxRows]; b.cellBlocks[i].reset(array, 0, array.length); } return b; }
return GTInfo.builder().setCodeSystem(codeSystem).//
public boolean isUsingDatetimeEncoding(int index) { return info.getCodeSystem().getDimEnc(index) instanceof AbstractDateDimEnc; }
private ByteArray encodeTime(long ts, int index, int roundingFlag) { String value; DataType partitionColType = info.getColumnType(index); if (partitionColType.isDate()) { value = DateFormat.formatToDateStr(ts); } else if (partitionColType.isTimeFamily()) { value = DateFormat.formatToTimeWithoutMilliStr(ts); } else if (partitionColType.isStringFamily() || partitionColType.isIntegerFamily()) {//integer like 20160101 String partitionDateFormat = segment.getModel().getPartitionDesc().getPartitionDateFormat(); if (StringUtils.isEmpty(partitionDateFormat)) { value = "" + ts; } else { value = DateFormat.formatToDateStr(ts, partitionDateFormat); } } else { throw new RuntimeException("Type " + partitionColType + " is not valid partition column type"); } ByteBuffer buffer = ByteBuffer.allocate(info.getMaxColumnLength()); info.getCodeSystem().encodeColumnValue(index, value, roundingFlag, buffer); return ByteArray.copyOf(buffer.array(), 0, buffer.position()); } }
public static ByteArray enc(GTInfo info, int col, String value) { ByteBuffer buf = ByteBuffer.allocate(info.getMaxColumnLength()); info.getCodeSystem().encodeColumnValue(col, value, buf); return ByteArray.copyOf(buf.array(), buf.arrayOffset(), buf.position()); }
public static CompareTupleFilter getCompareTupleFilter(int col, Object value) { TblColRef colRef = gtInfo.colRef(col); ColumnTupleFilter colFilter = new ColumnTupleFilter(colRef); ByteArray space = new ByteArray(gtInfo.getCodeSystem().maxCodeLength(col)); gtInfo.getCodeSystem().encodeColumnValue(col, value, space.asBuffer()); ConstantTupleFilter constFilter = new ConstantTupleFilter(space); CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ); compareFilter.addChild(colFilter); compareFilter.addChild(constFilter); return compareFilter; }
public int getMaxRecordLength() { return getMaxColumnLength(colAll); }
Writer(long startOffset) { this.writeOffset = startOffset; this.buf = ByteBuffer.allocate(info.getMaxRecordLength()); if (debug) logger.debug(ConcurrentDiskStore.this + " write start @ " + writeOffset); OutputStream out = new OutputStream() { byte[] tmp = new byte[1]; @Override public void write(int b) throws IOException { tmp[0] = (byte) b; write(tmp, 0, 1); } @Override public void write(byte[] bytes, int offset, int length) throws IOException { while (length > 0) { int n = writeChannel.write(ByteBuffer.wrap(bytes, offset, length), writeOffset); offset += n; length -= n; writeOffset += n; } } }; dout = new DataOutputStream(new BufferedOutputStream(out, STREAM_BUFFER_SIZE)); }
public static void prepareTestData() throws Exception { try { util.getHBaseAdmin().disableTable(TABLE); util.getHBaseAdmin().deleteTable(TABLE); } catch (Exception e) { // ignore table not found } Table table = util.createTable(TABLE, FAM); HRegionInfo hRegionInfo = new HRegionInfo(table.getName()); region = util.createLocalHRegion(hRegionInfo, table.getTableDescriptor()); gtInfo = newInfo(); GridTable gridTable = newTable(gtInfo); IGTScanner scanner = gridTable.scan(new GTScanRequestBuilder().setInfo(gtInfo).setRanges(null) .setDimensions(null).setFilterPushDown(null).createGTScanRequest()); for (GTRecord record : scanner) { byte[] value = record.exportColumns(gtInfo.getPrimaryKey()).toBytes(); byte[] key = new byte[RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN + value.length]; System.arraycopy(Bytes.toBytes(baseCuboid), 0, key, RowConstants.ROWKEY_SHARDID_LEN, RowConstants.ROWKEY_CUBOIDID_LEN); System.arraycopy(value, 0, key, RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN, value.length); Put put = new Put(key); put.addColumn(FAM, COL_M, record.exportColumns(gtInfo.getColumnBlock(1)).toBytes()); region.put(put); } }
private void compareTwoGTInfo(GTInfo info, GTInfo sInfo) { Assert.assertEquals(info.tableName, sInfo.tableName); Assert.assertEquals(info.primaryKey, sInfo.primaryKey); for (int i = 0; i < info.colTypes.length; i++) { Assert.assertEquals(info.codeSystem.maxCodeLength(i), sInfo.codeSystem.maxCodeLength(i)); Assert.assertTrue(info.codeSystem.maxCodeLength(i) > 0); Assert.assertEquals(info.colRef(i), sInfo.colRef(i)); } Assert.assertArrayEquals(info.colBlocks, sInfo.colBlocks); Assert.assertEquals(info.getRowBlockSize(), sInfo.getRowBlockSize()); Assert.assertEquals(info.rowBlockSize, sInfo.rowBlockSize); } }
public BufferedMeasureCodec createMeasureCodec() { DataType[] metricTypes = new DataType[aggrMetrics.trueBitCount()]; for (int i = 0; i < metricTypes.length; i++) { metricTypes[i] = info.getColumnType(aggrMetrics.trueBitAt(i)); } BufferedMeasureCodec codec = new BufferedMeasureCodec(metricTypes); codec.setBufferSize(info.getMaxColumnLength(aggrMetrics)); return codec; }
public GTScannerBenchmark() { Builder builder = GTInfo.builder(); builder.setCodeSystem(new GTSampleCodeSystem()); DataType tint = DataType.getType("int4"); DataType tlong = DataType.getType("long8"); builder.setColumns(tint, tint, tint, tint, tint, tlong, tlong); builder.setPrimaryKey(ImmutableBitSet.valueOf(0, 1, 2, 3, 4)); info = builder.build(); gen = new SortedGTRecordGenerator(info); gen.addDimension(10, 4, null); gen.addDimension(10, 4, null); gen.addDimension(10, 4, null); gen.addDimension(10, 4, null); gen.addDimension(100, 4, null); gen.addMeasure(8); gen.addMeasure(8); // warm up long t = System.currentTimeMillis(); testGenerate(); genTime = System.currentTimeMillis() - t; }
public boolean satisfyNone() { if (valueSet != null) { return valueSet.isEmpty(); } else if (begin.array() != null && end.array() != null) { return gtInfo.getCodeSystem().getComparator().compare(begin, end) > 0; } else { return false; } }
private ByteArray encodeTime(long ts, int index, int roundingFlag) { String value; DataType partitionColType = info.getColumnType(index); if (partitionColType.isDate()) { value = DateFormat.formatToDateStr(ts); } else if (partitionColType.isTimeFamily()) { value = DateFormat.formatToTimeWithoutMilliStr(ts); } else if (partitionColType.isStringFamily() || partitionColType.isIntegerFamily()) {//integer like 20160101 String partitionDateFormat = segment.getModel().getPartitionDesc().getPartitionDateFormat(); if (StringUtils.isEmpty(partitionDateFormat)) { value = "" + ts; } else { value = DateFormat.formatToDateStr(ts, partitionDateFormat); } } else { throw new RuntimeException("Type " + partitionColType + " is not valid partition column type"); } ByteBuffer buffer = ByteBuffer.allocate(info.getMaxColumnLength()); info.getCodeSystem().encodeColumnValue(index, value, roundingFlag, buffer); return ByteArray.copyOf(buffer.array(), 0, buffer.position()); } }
public GTConvertDecorator(Set<TblColRef> unevaluatableColumnCollector, Map<TblColRef, Integer> colMapping, GTInfo info, boolean encodeConstants) { this.unevaluatableColumnCollector = unevaluatableColumnCollector; this.colMapping = colMapping; this.info = info; this.useEncodeConstants = encodeConstants; buf = ByteBuffer.allocate(info.getMaxColumnLength()); }