private void readMinMax(DataInput in) throws IOException { DataType dataType = columnSpec.getSchemaDataType(); if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) { this.setMaxValue(in.readByte());
private void writeMinMax(DataOutput out) throws IOException { DataType dataType = columnSpec.getSchemaDataType(); if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) { out.writeByte((byte) getMaxValue()); out.writeFloat(0f); // unique value is obsoleted, maintain for compatibility } else if (DataTypes.isDecimal(dataType)) { byte[] maxAsBytes = getMaxAsBytes(columnSpec.getSchemaDataType()); byte[] minAsBytes = getMinAsBytes(columnSpec.getSchemaDataType()); byte[] unique = DataTypeUtil.bigDecimalToByte(BigDecimal.ZERO); out.writeShort((short) maxAsBytes.length);
@Override public ColumnPage decode(byte[] input, int offset, int length) throws MemoryException, IOException { DataType dataType = columnSpec.getSchemaDataType(); DataInputStream in = new DataInputStream(new ByteArrayInputStream(input, offset, length)); ColumnPage resultPage = ColumnPage.newPage( new ColumnPageEncoderMeta(columnSpec, dataType, compressorName), pageSize); if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) { decodeBytePage(in, resultPage); } else if (dataType == DataTypes.SHORT) { decodeShortPage(in, resultPage); } else if (dataType == DataTypes.INT) { decodeIntPage(in, resultPage); } else if (dataType == DataTypes.LONG) { decodeLongPage(in, resultPage); } else { throw new RuntimeException("unsupported datatype:" + dataType); } return resultPage; }
if (columnPageEncoderMeta.getColumnSpec().getSchemaDataType() == DataTypes.BOOLEAN) { value = BooleanConvert.boolean2Byte((Boolean) value);
private byte[] getChunkDataInBytes(int rowId) { ColumnType columnType = columnPage.getColumnSpec().getColumnType(); DataType srcDataType = columnPage.getColumnSpec().getSchemaDataType(); DataType targetDataType = columnPage.getDataType(); if (null != localDictionary) {
if (isUnsafeEnabled(meta)) { page = new UnsafeDecimalColumnPage( new ColumnPageEncoderMeta(columnSpec, columnSpec.getSchemaDataType(), compressorName), rowId); } else { page = new SafeDecimalColumnPage( new ColumnPageEncoderMeta(columnSpec, columnSpec.getSchemaDataType(), compressorName), rowId);
} else if (i < noDictionaryCount) { if (DataTypeUtil .isPrimitiveColumn(noDictDimensionPages[i].getColumnSpec().getSchemaDataType())) {
/** * method to add complex column data * @param depth * complexColumnIndex of column * @param dataList * dataList */ public void putComplexData(int depth, List<byte[]> dataList) { assert (depth <= this.complexColumnIndex); int positionNumber = currentRowIdList[depth]; for (byte[] value : dataList) { if (columnPages[depth].getDataType() != DataTypes.BYTE_ARRAY) { if ((value == null) || (value.length == 0)) { columnPages[depth].putNull(positionNumber); columnPages[depth].statsCollector.updateNull(positionNumber); columnPages[depth].nullBitSet.set(positionNumber); } else { columnPages[depth].putData(positionNumber, DataTypeUtil .getDataBasedOnDataTypeForNoDictionaryColumn(value, columnPages[depth].getColumnSpec().getSchemaDataType(), false)); } } else { columnPages[depth].putData(positionNumber, value); } positionNumber++; } currentRowIdList[depth] = positionNumber; }
if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) { byte value = getByte(rowId); if (columnPageEncoderMeta.getColumnSpec().getSchemaDataType() == DataTypes.BOOLEAN) { return BooleanConvert.byte2Boolean(value);
private Object getValue(ColumnPage page, int rowId) { DataType type = page.getColumnSpec().getSchemaDataType(); Object value = null; if (type == DataTypes.BYTE) {
/** * Get null at rowId */ private Object getNull(int rowId) { Object result; DataType dataType = columnPageEncoderMeta.getStoreDataType(); if (dataType == DataTypes.BOOLEAN) { result = getBoolean(rowId); } else if (dataType == DataTypes.BYTE) { result = getByte(rowId); if (columnPageEncoderMeta.getColumnSpec().getSchemaDataType() == DataTypes.BOOLEAN) { result = BooleanConvert.byte2Boolean((byte)result); } } else if (dataType == DataTypes.SHORT) { result = getShort(rowId); } else if (dataType == DataTypes.INT) { result = getInt(rowId); } else if (dataType == DataTypes.LONG) { result = getLong(rowId); } else if (dataType == DataTypes.DOUBLE) { result = getDouble(rowId); } else if (DataTypes.isDecimal(dataType)) { result = getDecimal(rowId); } else { throw new IllegalArgumentException("unsupported data type: " + dataType); } return result; }
public static EncodedColumnPage encodedColumn(ColumnPage page) throws IOException, MemoryException { ColumnPageEncoder pageEncoder = createCodecForDimension(page); if (pageEncoder == null) { ColumnPageEncoder encoder = new DirectCompressCodec(DataTypes.BYTE_ARRAY).createEncoder(null); return encoder.encode(page); } else { LOGGER.debug("Encoder result ---> Source data type: " + pageEncoder.getEncoderMeta(page) .getColumnSpec().getSchemaDataType() + " Destination data type: " + pageEncoder .getEncoderMeta(page).getStoreDataType() + " for the column: " + pageEncoder .getEncoderMeta(page).getColumnSpec().getFieldName()); return pageEncoder.encode(page); } }
public DataType getSchemaDataType() { return columnSpec.getSchemaDataType(); }
@Override public ColumnPageEncoder createEncoder(TableSpec.ColumnSpec columnSpec, ColumnPage inputPage) { // TODO: add log // choose the encoding type for measure type and no dictionary primitive type columns if (columnSpec instanceof TableSpec.MeasureSpec || ( DataTypeUtil.isPrimitiveColumn(columnSpec.getSchemaDataType()) && columnSpec.getColumnType() == ColumnType.PLAIN_VALUE)) { return createEncoderForMeasureOrNoDictionaryPrimitive(inputPage, columnSpec); } else { if (newWay) { return createEncoderForDimension((TableSpec.DimensionSpec) columnSpec, inputPage); } else { assert columnSpec instanceof TableSpec.DimensionSpec; return createEncoderForDimensionLegacy((TableSpec.DimensionSpec) columnSpec); } } }
private static ColumnPage newFixedByteArrayPage(TableSpec.ColumnSpec columnSpec, byte[] lvEncodedByteArray, int eachValueSize, String compressorName) throws MemoryException { int pageSize = lvEncodedByteArray.length / eachValueSize; ColumnPage fixLengthByteArrayPage = createFixLengthByteArrayPage( new ColumnPageEncoderMeta(columnSpec, columnSpec.getSchemaDataType(), compressorName), pageSize, eachValueSize); byte[] data = null; int offset = 0; for (int i = 0; i < pageSize; i++) { data = new byte[eachValueSize]; System.arraycopy(lvEncodedByteArray, offset, data, 0, eachValueSize); fixLengthByteArrayPage.putBytes(i, data); offset += eachValueSize; } return fixLengthByteArrayPage; }
private RLEDecoder(TableSpec.ColumnSpec columnSpec, int pageSize, String compressorName) { validateDataType(columnSpec.getSchemaDataType()); this.columnSpec = columnSpec; this.pageSize = pageSize; this.compressorName = compressorName; }
private static ColumnPage newDecimalPage(ColumnPageEncoderMeta meta, byte[][] byteArray) { ColumnPageEncoderMeta encoderMeta = new ColumnPageEncoderMeta(meta.getColumnSpec(), meta.getColumnSpec().getSchemaDataType(), meta.getCompressorName()); encoderMeta.setFillCompleteVector(meta.isFillCompleteVector()); ColumnPage columnPage = createPage( encoderMeta, byteArray.length); columnPage.setByteArrayPage(byteArray); return columnPage; }
public int getPrecision() { if (DataTypes.isDecimal(columnSpec.getSchemaDataType())) { return columnSpec.getPrecision(); } throw new UnsupportedOperationException(); }
public int getScale() { if (DataTypes.isDecimal(columnSpec.getSchemaDataType())) { return columnSpec.getScale(); } throw new UnsupportedOperationException(); }