public static byte[] toXorBytes(long val) { val = val ^ Long.MIN_VALUE; return toBytes(val); }
public static byte[] toXorBytes(int val) { val = val ^ Integer.MIN_VALUE; return toBytes(val); }
/** * If number type column is in sort_columns, the column will be no-dictionary column. * It will compare byte arrays to sort the data. * For example the binary string of int value as follows. * 1 : 00000000 00000000 00000000 00000001 * -1 : 11111111 11111111 11111111 11111111 * In this case, the compare method of byte arrays will return a wrong result.(1 < -1) * The root cause is that the sign bit of negative number is 1. * These XOR methods will change the sign bit as follows. * 1 ^ MIN_VALUE : 10000000 00000000 00000000 00000001 * -1 ^ MIN_VALUE : 01111111 11111111 11111111 11111111 * After the transform, the compare method of byte arrays will return a right result.(1 > -1) */ public static byte[] toXorBytes(short val) { val = (short) (val ^ Short.MIN_VALUE); return toBytes(val); }
public static byte[] toBytes(double val) { return toBytes(Double.doubleToLongBits(val)); }
public static byte[] getBytesDataDataTypeForNoDictionaryColumn(Object dimensionValue, DataType actualDataType) { if (dimensionValue == null) { if (actualDataType == DataTypes.STRING) { return CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY; } else { return new byte[0]; } } if (actualDataType == DataTypes.BOOLEAN) { return ByteUtil.toBytes((Boolean) dimensionValue); } else if (actualDataType == DataTypes.SHORT) { return ByteUtil.toXorBytes((Short) dimensionValue); } else if (actualDataType == DataTypes.INT) { return ByteUtil.toXorBytes((Integer) dimensionValue); } else if (actualDataType == DataTypes.LONG) { return ByteUtil.toXorBytes((Long) dimensionValue); } else if (actualDataType == DataTypes.TIMESTAMP) { return ByteUtil.toXorBytes((Long)dimensionValue); } else { // Default action for String/Varchar return ByteUtil.toBytes(dimensionValue.toString()); } }
public RangePartitioner(PartitionInfo partitionInfo) { List<String> values = partitionInfo.getRangeInfo(); DataType partitionColumnDataType = partitionInfo.getColumnSchemaList().get(0).getDataType(); numPartitions = values.size(); bounds = new Object[numPartitions]; if (partitionColumnDataType == DataTypes.STRING) { for (int i = 0; i < numPartitions; i++) { bounds[i] = ByteUtil.toBytes(values.get(i)); } } else { for (int i = 0; i < numPartitions; i++) { bounds[i] = PartitionUtil.getDataBasedOnDataType(values.get(i), partitionColumnDataType, timestampFormatter, dateFormatter); } } comparator = Comparator.getComparator(partitionColumnDataType); }
public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensionValue, DataType actualDataType, String dateFormat) { if (actualDataType == DataTypes.BOOLEAN) { return ByteUtil.toBytes(BooleanConvert.parseBoolean(dimensionValue)); } else if (actualDataType == DataTypes.SHORT) { return ByteUtil.toXorBytes(Short.parseShort(dimensionValue)); return ByteUtil.toBytes(dimensionValue);
timestampFormatter, dateFormatter); if (listValue instanceof String) { listValue = ByteUtil.toBytes((String)listValue); timestampFormatter, dateFormatter); if (listValue instanceof String) { listValue = ByteUtil.toBytes((String)listValue); timestampFormatter, dateFormatter); if (listValue instanceof String) { listValue = ByteUtil.toBytes((String)listValue); timestampFormatter, dateFormatter); if (listValue instanceof String) { listValue = ByteUtil.toBytes((String)listValue);
@Override public BitSet applyFilter(Partitioner partitioner) { BitSet partitionMap = PartitionUtil.generateBitSetBySize(partitioner.numPartitions(), false); if (equalTo.isNull) { partitionMap.set(partitioner.getPartition(null)); } else { LiteralExpression literal = (LiteralExpression) equalTo.getRight(); Object value = PartitionUtil.getDataBasedOnDataTypeForFilter( literal.getLiteralExpValue().toString(), partitionInfo.getColumnSchemaList().get(0).getDataType()); if (PartitionType.RANGE == partitionInfo.getPartitionType() && value instanceof String) { value = ByteUtil.toBytes((String)value); } partitionMap.set(partitioner.getPartition(value)); } return partitionMap; } }
values.get(partitionIndex), partitionColumnDataType, timestampFormatter, dateFormatter); if (value instanceof String) { value = ByteUtil.toBytes((String)value);
@Override public BitSet applyFilter(Partitioner partitioner) { BitSet partitionMap = PartitionUtil.generateBitSetBySize(partitioner.numPartitions(), false); ListExpression list = (ListExpression) in.getRight(); for (Expression expr : list.getChildren()) { LiteralExpression literal = (LiteralExpression) expr; Object value = PartitionUtil.getDataBasedOnDataTypeForFilter( literal.getLiteralExpValue().toString(), partitionInfo.getColumnSchemaList().get(0).getDataType()); if (PartitionType.RANGE == partitionInfo.getPartitionType() && value instanceof String) { value = ByteUtil.toBytes((String)value); } partitionMap.set(partitioner.getPartition(value)); } return partitionMap; } }
partitionInfo.getColumnSchemaList().get(0).getDataType()); if (filterValueOfList instanceof String) { filterValueOfList = ByteUtil.toBytes((String)filterValueOfList); partitionInfo.getColumnSchemaList().get(0).getDataType()); if (filterValueOfRange instanceof String) { filterValueOfRange = ByteUtil.toBytes((String)filterValueOfRange);
} else if (srcDataType == DataTypes.BOOLEAN) { byte out = (byte) longData; return ByteUtil.toBytes(ByteUtil.toBoolean(out)); } else if (srcDataType == DataTypes.SHORT) { short out = (short) longData; byte[] out = new byte[1]; out[0] = (columnPage.getByte(rowId)); return ByteUtil.toBytes(ByteUtil.toBoolean(out)); } else if (srcDataType == DataTypes.BYTE_ARRAY) { return columnPage.getBytes(rowId);
} else { output.writeInt((int) columnValue); output.dimStatsCollectors[dimCount].update(ByteUtil.toBytes((int) columnValue));