private int buildKey(Cuboid cuboid, ByteArray[] splitBuffers) { RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(cuboid); int startIdx = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboidId int endIdx = startIdx + Long.bitCount(cuboid.getId()); int offset = 0; for (int i = startIdx; i < endIdx; i++) { System.arraycopy(splitBuffers[i].array(), splitBuffers[i].offset(), newKeyBodyBuf, offset, splitBuffers[i].length()); offset += splitBuffers[i].length(); } int fullKeySize = rowkeyEncoder.getBytesLength(); while (newKeyBuf.array().length < fullKeySize) { newKeyBuf = new ByteArray(newKeyBuf.length() * 2); } newKeyBuf.setLength(fullKeySize); rowkeyEncoder.encode(new ByteArray(newKeyBodyBuf, 0, offset), newKeyBuf); return fullKeySize; }
@Override public void aggregate(ByteArray value) { if (byteArray == null) { byteArray = value; } else { if (!byteArray.equals(value)) { if (!warned) { logger.warn("Extended column must be unique given same host column"); warned = true; } } } }
@Override public List<ByteArray> valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { if (values.length != 1) throw new IllegalArgumentException(); //source input column value String literal = values[0]; // encode literal using dictionary TblColRef literalCol = getRawColumn(measureDesc.getFunction()); Dictionary<String> dictionary = dictionaryMap.get(literalCol); int keyEncodedValue = dictionary.getIdFromValue(literal); ByteArray key = new ByteArray(dictionary.getSizeOfId()); BytesUtil.writeUnsigned(keyEncodedValue, key.array(), key.offset(), dictionary.getSizeOfId()); List<ByteArray> valueList = new ArrayList<ByteArray>(1); valueList.add(key); return valueList; }
public CubeStatsResult(Path path, int precision) throws IOException { Configuration hadoopConf = HadoopUtil.getCurrentConfiguration(); Option seqInput = SequenceFile.Reader.file(path); try (Reader reader = new SequenceFile.Reader(hadoopConf, seqInput)) { LongWritable key = (LongWritable) ReflectionUtils.newInstance(reader.getKeyClass(), hadoopConf); BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), hadoopConf); while (reader.next(key, value)) { if (key.get() == 0L) { percentage = Bytes.toInt(value.getBytes()); } else if (key.get() == -1) { mapperOverlapRatio = Bytes.toDouble(value.getBytes()); } else if (key.get() == -2) { mapperNumber = Bytes.toInt(value.getBytes()); } else if (key.get() == -3) { sourceRecordCount = Bytes.toLong(value.getBytes()); } else if (key.get() > 0) { HLLCounter hll = new HLLCounter(precision); ByteArray byteArray = new ByteArray(value.getBytes()); hll.readRegisters(byteArray.asBuffer()); counterMap.put(key.get(), hll); } } } }
private List<ByteArray> getValueList(int size) { if (size == -1) { return null; } List<ByteArray> valueList = new ArrayList<ByteArray>(size); for (Integer i = 0; i < size; i++) { ByteArray key = new ByteArray(1); BytesUtil.writeUnsigned(i, key.array(), 0, key.length()); valueList.add(key); } return valueList; }
public void testWriteReadUnsignedInt(int testInt, int length) { ByteArray ba = new ByteArray(new byte[length]); BytesUtil.writeUnsigned(testInt, length, ba.asBuffer()); byte[] newBytes = new byte[length]; System.arraycopy(ba.array(), 0, newBytes, 0, length); int value = BytesUtil.readUnsigned(new ByteArray(newBytes).asBuffer(), length); assertEquals(value, testInt); byte[] anOtherNewBytes = new byte[length]; BytesUtil.writeUnsigned(testInt, anOtherNewBytes, 0, length); assertTrue(Arrays.equals(anOtherNewBytes, ba.array())); }
@Override public List<ByteArray> deserialize(ByteBuffer in) { List<ByteArray> values = new ArrayList<>(); int size = BytesUtil.readVInt(in); if (size >= 0) { for (int i = 0; i < size; i++) { ByteArray ba = new ByteArray(BytesUtil.readByteArray(in)); if (ba.length() != 0) { values.add(ba); } } } else { throw new RuntimeException("Read error data size:" + size); } return values; } }
@Test public void testOverflow() { String text = StringUtils.repeat("h", 21); ExtendedColumnSerializer serializer = new ExtendedColumnSerializer(DataType.getType("extendedcolumn(20)")); MeasureIngester<ByteArray> ingester = measureType.newIngester(); ByteArray array = ingester.valueOf(new String[] { null, text }, null, null); ByteBuffer buffer = ByteBuffer.allocate(serializer.maxLength()); serializer.serialize(array, buffer); buffer.flip(); ByteArray des = serializer.deserialize(buffer); Assert.assertTrue(new ByteArray(StringUtils.repeat("h", 20).getBytes(StandardCharsets.UTF_8)).equals(des)); } }
@Override public int compare(T o1, T o2) { //return BytesUtil.safeCompareBytes(converter.convertToBytes(o1),converter.convertToBytes(o2)); byte[] b1 = converter.convertToBytes(o1); byte[] b2 = converter.convertToBytes(o2); ByteArray ba1 = new ByteArray(b1, 0, b1.length); ByteArray ba2 = new ByteArray(b2, 0, b2.length); return ba1.compareTo(ba2); } }
@Override public void dump(PrintStream out) { out.println("TrieDictionaryForest"); out.println("baseId:" + baseId); StringBuilder sb = new StringBuilder(); sb.append("value divide:"); for (ByteArray ba : valueDivide) sb.append(bytesConvert.convertFromBytes(ba.array(), 0, ba.length()) + " "); sb.append("\noffset divide:"); for (Integer offset : accuOffset) sb.append(offset + " "); out.println(sb.toString()); for (int i = 0; i < trees.size(); i++) { out.println("----tree " + i + "--------"); trees.get(i).dump(out); } }
Writer() { cellBlockBuffers = new ByteBuffer[info.colBlocks.length]; for (int i = 0; i < cellBlockBuffers.length; i++) { cellBlockBuffers[i] = cellBlocks[i].asBuffer(); } }
@Override public int getMemBytesEstimate() { int bytes = 0; if (list != null) { for (ByteArray array : list) { bytes += array.length() + 1; } } return bytes; }