/** * @return the dataType */ public DataType getDataType() { return columnSchema.getDataType(); }
/** * @return if DataType is ARRAY or STRUCT, this method return true, else * false. */ public Boolean isComplex() { return columnSchema.getDataType().isComplexType(); }
public ListPartitioner(PartitionInfo partitionInfo) { List<List<String>> values = partitionInfo.getListInfo(); DataType partitionColumnDataType = partitionInfo.getColumnSchemaList().get(0).getDataType(); numPartitions = values.size(); for (int i = 0; i < numPartitions; i++) { for (String value : values.get(i)) { map.put(PartitionUtil.getDataBasedOnDataType(value, partitionColumnDataType, timestampFormatter, dateFormatter), i + 1); } } }
public HashPartitionerImpl(List<Integer> indexes, List<ColumnSchema> columnSchemas, int numberOfBuckets) { this.numberOfBuckets = numberOfBuckets; hashes = new Hash[indexes.size()]; for (int i = 0; i < indexes.size(); i++) { DataType dataType = columnSchemas.get(i).getDataType(); if (dataType == DataTypes.SHORT || dataType == DataTypes.INT || dataType == DataTypes.LONG) { hashes[i] = new IntegralHash(indexes.get(i)); } else if (dataType == DataTypes.DOUBLE || dataType == DataTypes.FLOAT || DataTypes.isDecimal(dataType)) { hashes[i] = new DecimalHash(indexes.get(i)); } else { hashes[i] = new StringHash(indexes.get(i)); } } }
/** * This method will parse a given string value corresponding to its data type * * @param value value to parse * @param columnSchema dimension to get data type and precision and scale in case of decimal * data type * @return */ public static String normalizeColumnValueForItsDataType(String value, ColumnSchema columnSchema) { try { Object parsedValue = null; DataType dataType = columnSchema.getDataType(); if (DataTypes.isDecimal(dataType)) { return parseStringToBigDecimal(value, columnSchema); } else if (dataType == DataTypes.SHORT || dataType == DataTypes.INT || dataType == DataTypes.LONG) { parsedValue = normalizeIntAndLongValues(value, columnSchema.getDataType()); } else if (dataType == DataTypes.DOUBLE) { parsedValue = Double.parseDouble(value); } else { return value; } if (null != parsedValue) { return value; } return null; } catch (Exception e) { return null; } }
if (!isDefaultValueNull(defaultValue)) { String value; DataType dataType = columnSchema.getDataType(); if (dataType == DataTypes.SHORT) { value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
if (!isDefaultValueNull(defaultValue)) { String value; DataType dataType = columnSchema.getDataType(); if (dataType == DataTypes.SHORT) { value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
public RangePartitioner(PartitionInfo partitionInfo) { List<String> values = partitionInfo.getRangeInfo(); DataType partitionColumnDataType = partitionInfo.getColumnSchemaList().get(0).getDataType(); numPartitions = values.size(); bounds = new Object[numPartitions]; if (partitionColumnDataType == DataTypes.STRING) { for (int i = 0; i < numPartitions; i++) { bounds[i] = ByteUtil.toBytes(values.get(i)); } } else { for (int i = 0; i < numPartitions; i++) { bounds[i] = PartitionUtil.getDataBasedOnDataType(values.get(i), partitionColumnDataType, timestampFormatter, dateFormatter); } } comparator = Comparator.getComparator(partitionColumnDataType); }
/** * Construct Field from ColumnSchema * * @param columnSchema ColumnSchema, Store the information about the column meta data */ public Field(ColumnSchema columnSchema) { this.name = columnSchema.getColumnName(); this.type = columnSchema.getDataType(); children = new LinkedList<>(); schemaOrdinal = columnSchema.getSchemaOrdinal(); precision = columnSchema.getPrecision(); scale = columnSchema.getScale(); }
for (int index = 0; index < allColumns.size(); index++) { ColumnSchema colSchema = allColumns.get(index); if (colSchema.getDataType() == DataTypes.STRING || colSchema.getDataType() == DataTypes.VARCHAR) { colSchema.setLocalDictColumn(true); allColumns.set(index, colSchema);
/** * Preparing the boolean [] to map whether the dimension is varchar data type or not. */ public static boolean[] getIsVarcharColumnMapping(DataField[] fields) { List<Boolean> isVarcharColumnMapping = new ArrayList<Boolean>(); for (DataField field : fields) { // for complex type need to break the loop if (field.getColumn().isComplex()) { break; } if (field.getColumn().isDimension()) { isVarcharColumnMapping.add( field.getColumn().getColumnSchema().getDataType() == DataTypes.VARCHAR); } } return ArrayUtils.toPrimitive( isVarcharColumnMapping.toArray(new Boolean[isVarcharColumnMapping.size()])); }
columnLocalDictGenMap.put(columnSchema.getColumnName(), new ColumnLocalDictionaryGenerator(localDictionaryThreshold, columnSchema.getDataType() == DataTypes.VARCHAR ? CarbonCommonConstants.INT_SIZE_IN_BYTE : CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
/** * traverse through the columns of complex column specified in local dictionary include, * and set local dictionary for all the string and varchar child columns * @param allColumns * @param dimensionOrdinal * @param childColumnCount * @return */ private static int setLocalDictForComplexColumns(List<ColumnSchema> allColumns, int dimensionOrdinal, int childColumnCount) { for (int i = 0; i < childColumnCount; i++) { ColumnSchema column = allColumns.get(dimensionOrdinal); if (column.getNumberOfChild() > 0) { dimensionOrdinal++; setLocalDictForComplexColumns(allColumns, dimensionOrdinal, column.getNumberOfChild()); } else { if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) || column.getDataType().equals(DataTypes.VARCHAR)) && !column.hasEncoding(Encoding.DICTIONARY)) { column.setLocalDictColumn(true); } } dimensionOrdinal++; } return dimensionOrdinal; }
@Override public BitSet applyFilter(Partitioner partitioner) { BitSet partitionMap = PartitionUtil.generateBitSetBySize(partitioner.numPartitions(), false); if (equalTo.isNull) { partitionMap.set(partitioner.getPartition(null)); } else { LiteralExpression literal = (LiteralExpression) equalTo.getRight(); Object value = PartitionUtil.getDataBasedOnDataTypeForFilter( literal.getLiteralExpValue().toString(), partitionInfo.getColumnSchemaList().get(0).getDataType()); if (PartitionType.RANGE == partitionInfo.getPartitionType() && value instanceof String) { value = ByteUtil.toBytes((String)value); } partitionMap.set(partitioner.getPartition(value)); } return partitionMap; } }
@Override public BitSet applyFilter(Partitioner partitioner) { BitSet partitionMap = PartitionUtil.generateBitSetBySize(partitioner.numPartitions(), false); ListExpression list = (ListExpression) in.getRight(); for (Expression expr : list.getChildren()) { LiteralExpression literal = (LiteralExpression) expr; Object value = PartitionUtil.getDataBasedOnDataTypeForFilter( literal.getLiteralExpValue().toString(), partitionInfo.getColumnSchemaList().get(0).getDataType()); if (PartitionType.RANGE == partitionInfo.getPartitionType() && value instanceof String) { value = ByteUtil.toBytes((String)value); } partitionMap.set(partitioner.getPartition(value)); } return partitionMap; } }
if (dimension.getColumnSchema().getDataType() == DataTypes.VARCHAR) { isVarcharDimMapping[i] = true;
Object filterValueOfList = PartitionUtil.getDataBasedOnDataTypeForFilter( literal.getLiteralExpValue().toString(), partitionInfo.getColumnSchemaList().get(0).getDataType()); if (filterValueOfList instanceof String) { filterValueOfList = ByteUtil.toBytes((String)filterValueOfList); Object filterValueOfRange = PartitionUtil.getDataBasedOnDataTypeForFilter( literal.getLiteralExpValue().toString(), partitionInfo.getColumnSchemaList().get(0).getDataType()); if (filterValueOfRange instanceof String) { filterValueOfRange = ByteUtil.toBytes((String)filterValueOfRange);
DataType dataType = dataFileFooter.getColumnInTable().get(i).getDataType(); SerializableComparator comparator = Comparator.getComparator(dataType); int compare;
for (int i = 0; i < carbonColumns.length; i++) { ColumnSchema carbonColumn = carbonColumns[i]; DataType dataType = carbonColumn.getDataType(); if (org.apache.carbondata.core.metadata.datatype.DataTypes.isDecimal(dataType)) { fields.add(new StructField(carbonColumn.getColumnName(), } else { fields.add(new StructField(carbonColumn.getColumnName(), convertCarbonToSparkDataType(carbonColumn.getDataType()), true, Metadata.empty()));
private static void updateDecimalType(TableInfo tableInfo) { List<ColumnSchema> deserializedColumns = tableInfo.getFactTable().getListOfColumns(); for (ColumnSchema column : deserializedColumns) { DataType dataType = column.getDataType(); if (DataTypes.isDecimal(dataType)) { column.setDataType(DataTypes.createDecimalType(column.getPrecision(), column.getScale())); } } if (tableInfo.getFactTable().getPartitionInfo() != null) { List<ColumnSchema> partitionColumns = tableInfo.getFactTable().getPartitionInfo().getColumnSchemaList(); for (ColumnSchema column : partitionColumns) { DataType dataType = column.getDataType(); if (DataTypes.isDecimal(dataType)) { column.setDataType(DataTypes.createDecimalType(column.getPrecision(), column.getScale())); } } } }