/** * initialise data type for measures for their storage format */ public static DataType[] initDataType(CarbonTable carbonTable, String tableName, int measureCount) { DataType[] type = new DataType[measureCount]; for (int i = 0; i < type.length; i++) { type[i] = DataTypes.DOUBLE; } List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(tableName); for (int i = 0; i < measureCount; i++) { type[i] = measures.get(i).getDataType(); } return type; }
private void addMeasures(List<CarbonMeasure> measures) { for (int i = 0; i < measures.size(); i++) { CarbonMeasure measure = measures.get(i); measureSpec[i] = new MeasureSpec(measure.getColName(), measure.getDataType()); } }
public int[] getColumnsValueSize() { int[] dimensionValueSize = new int[eachDimColumnValueSize.length + eachComplexDimColumnValueSize.length + measures .size()]; System .arraycopy(eachDimColumnValueSize, 0, dimensionValueSize, 0, eachDimColumnValueSize.length); System.arraycopy(eachComplexDimColumnValueSize, 0, dimensionValueSize, eachDimColumnValueSize.length, eachComplexDimColumnValueSize.length); int k = eachDimColumnValueSize.length + eachComplexDimColumnValueSize.length; for (int i = 0; i < measures.size(); i++) { DataType dataType = measures.get(i).getDataType(); if (DataTypes.isDecimal(dataType)) { dimensionValueSize[k++] = -1; } else { dimensionValueSize[k++] = 8; } } return dimensionValueSize; }
public static DataType[] getMeasureDataType(int measureCount, CarbonTable carbonTable) { DataType[] type = new DataType[measureCount]; for (int i = 0; i < type.length; i++) { type[i] = DataTypes.DOUBLE; } List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(carbonTable.getTableName()); for (int i = 0; i < type.length; i++) { type[i] = measures.get(i).getDataType(); } return type; }
for (int i = 0; i < measures.size(); i++) { buffer.rewind(); DataType dataType = measures.get(i).getDataType(); if (dataType == DataTypes.BYTE) { buffer.putLong(Byte.MIN_VALUE);
for (int i = 0; i < measures.size(); i++) { buffer.rewind(); DataType dataType = measures.get(i).getDataType(); if (dataType == DataTypes.BYTE) { buffer.putLong(Byte.MAX_VALUE);
public RowLevelRangeLessThanEqualFilterExecuterImpl( List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList, List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp, AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues, Object[] msrFilterRangeValues, SegmentProperties segmentProperties) { super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties, null); this.filterRangeValues = filterRangeValues; this.msrFilterRangeValues = msrFilterRangeValues; if (!msrColEvalutorInfoList.isEmpty()) { CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); } ifDefaultValueMatchesFilter(); if (isDimensionPresentInCurrentBlock[0]) { isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex() && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn(); } }
public RowLevelRangeLessThanFilterExecuterImpl( List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList, List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp, AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues, Object[] msrFilterRangeValues, SegmentProperties segmentProperties) { super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties, null); this.filterRangeValues = filterRangeValues; this.msrFilterRangeValues = msrFilterRangeValues; if (!msrColEvalutorInfoList.isEmpty()) { CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); } ifDefaultValueMatchesFilter(); if (isDimensionPresentInCurrentBlock[0]) { isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex() && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn(); } }
RowLevelRangeGrtThanFiterExecuterImpl(List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList, List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp, AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues, Object[] msrFilterRangeValues, SegmentProperties segmentProperties) { super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties, null); this.filterRangeValues = filterRangeValues; this.msrFilterRangeValues = msrFilterRangeValues; if (!this.msrColEvalutorInfoList.isEmpty()) { CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); } if (isDimensionPresentInCurrentBlock[0]) { isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex() && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn(); } ifDefaultValueMatchesFilter(); }
RowLevelRangeGrtrThanEquaToFilterExecuterImpl( List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList, List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp, AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues, Object[] msrFilterRangeValues, SegmentProperties segmentProperties) { super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties, null); this.filterRangeValues = filterRangeValues; this.msrFilterRangeValues = msrFilterRangeValues; if (!msrColEvalutorInfoList.isEmpty()) { CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); } if (isDimensionPresentInCurrentBlock[0] == true) { isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex() && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn(); } ifDefaultValueMatchesFilter(); }
private List<Object> getMeasureRangeValues(CarbonMeasure carbonMeasure) { List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20); if (this.getFilterExpression() instanceof BinaryConditionalExpression) { listOfExpressionResults = ((BinaryConditionalExpression) this.getFilterExpression()).getLiterals(); } List<Object> filterValuesList = new ArrayList<>(20); boolean invalidRowsPresent = false; for (ExpressionResult result : listOfExpressionResults) { try { if (result.getString() == null) { filterValuesList.add(null); continue; } filterValuesList.add(DataTypeUtil.getMeasureValueBasedOnDataType(result.getString(), result.getDataType(), carbonMeasure.getScale(), carbonMeasure.getPrecision())); } catch (FilterIllegalMemberException e) { // Any invalid member while evaluation shall be ignored, system will log the // error only once since all rows the evaluation happens so inorder to avoid // too much log inforation only once the log will be printed. FilterUtil.logError(e, invalidRowsPresent); } } Collections.sort(filterValuesList, org.apache.carbondata.core.util.comparator.Comparator .getComparatorByDataTypeForMeasure(carbonMeasure.getDataType())); return filterValuesList; }
byte[] defaultValue = measure.getDefaultValue(); SerializableComparator comparatorTmp = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); if (null != defaultValue) { for (int k = 0; k < msrFilterRangeValues.length; k++) {
byte[] defaultValue = measure.getDefaultValue(); SerializableComparator comparatorTmp = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); if (null != defaultValue) { for (int k = 0; k < msrFilterRangeValues.length; k++) {
SerializableComparator comparatorTmp = (null != comparator ? comparator : Comparator.getComparatorByDataTypeForMeasure(measure.getDataType())); if (null != defaultValue) { for (int k = 0; k < msrFilterRangeValues.length; k++) {
byte[] defaultValue = measure.getDefaultValue(); SerializableComparator comparatorTmp = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); if (null != defaultValue) { for (int k = 0; k < msrFilterRangeValues.length; k++) {
if (keysBasedOnFilter[i] != null) { keysBasedOnFilter[i] = DataTypeUtil .getDataBasedOnDataType(keysBasedOnFilter[i].toString(), measures.getDataType(), converter);
Object getMeasureData(ColumnPage dataChunk, int index, CarbonMeasure carbonMeasure) { if (!dataChunk.getNullBits().get(index)) { DataType dataType = carbonMeasure.getDataType(); if (dataType == DataTypes.BOOLEAN) { return dataChunk.getBoolean(index); } else if (dataType == DataTypes.SHORT) { return (short) dataChunk.getLong(index); } else if (dataType == DataTypes.INT) { return (int) dataChunk.getLong(index); } else if (dataType == DataTypes.LONG) { return dataChunk.getLong(index); } else if (dataType == DataTypes.FLOAT) { return dataChunk.getFloat(index); } else if (dataType == DataTypes.BYTE) { return dataChunk.getByte(index); } else if (DataTypes.isDecimal(dataType)) { BigDecimal bigDecimalMsrValue = dataChunk.getDecimal(index); if (null != bigDecimalMsrValue && carbonMeasure.getScale() > bigDecimalMsrValue.scale()) { bigDecimalMsrValue = bigDecimalMsrValue.setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP); } // convert data type as per the computing engine return DataTypeUtil.getDataTypeConverter().convertFromBigDecimalToDecimal( bigDecimalMsrValue); } else { return dataChunk.getDouble(index); } } return null; }
CarbonMeasure measure = measureColumnResolvedFilterInfo.getMeasure(); SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); Object defaultValue = null; if (null != measure.getDefaultValue()) {
DataType dataType = msr.getMeasure().getDataType(); if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.SHORT || dataType == DataTypes.INT || dataType == DataTypes.LONG || dataType == DataTypes.FLOAT || dataType == DataTypes.BYTE) { fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), msr.getMeasure().getDataType()); } else if (DataTypes.isDecimal(dataType)) { fields[msr.getOrdinal()] = new StructField(msr.getColumnName(),
msrColResolvedFilterInfoCopyObject.setMeasure(measuresFromCurrentBlock); msrColResolvedFilterInfoCopyObject.setColumnIndex(measuresFromCurrentBlock.getOrdinal()); msrColResolvedFilterInfoCopyObject.setType(measuresFromCurrentBlock.getDataType()); return new ExcludeFilterExecuterImpl(null, msrColResolvedFilterInfoCopyObject, segmentProperties, true);