public static boolean[] getComplexDataTypeArray(ProjectionDimension[] queryDimensions) { boolean[] dictionaryEncodingArray = new boolean[queryDimensions.length]; for (int i = 0; i < queryDimensions.length; i++) { dictionaryEncodingArray[i] = queryDimensions[i].getDimension().getDataType().isComplexType(); } return dictionaryEncodingArray; }
/** * Get the no dictionary data types on the table * * @param carbonTable * @return */ public static DataType[] getNoDictDataTypes(CarbonTable carbonTable) { List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(carbonTable.getTableName()); List<DataType> type = new ArrayList<>(); for (int i = 0; i < dimensions.size(); i++) { if (dimensions.get(i).isSortColumn() && !dimensions.get(i).hasEncoding(Encoding.DICTIONARY)) { type.add(dimensions.get(i).getDataType()); } } return type.toArray(new DataType[type.size()]); }
/** * get dateformat * @param carbonDimension * @return */ private String getDateFormat(CarbonDimension carbonDimension) { String format; String dateFormat = null; if (this.carbonDimension.getDataType() == DataTypes.DATE) { dateFormat = carbonDimension.getDateFormat(); } if (dateFormat != null && !dateFormat.trim().isEmpty()) { format = dateFormat; } else { format = CarbonUtil.getFormatFromProperty(dataType); } return format; }
@Override public Object getDataBasedOnColumn(ByteBuffer dataBuffer, CarbonDimension parent, CarbonDimension child) { Object actualData; if (parent.getOrdinal() != child.getOrdinal() || null == dataBuffer || !dataBuffer .hasRemaining()) { return null; } int size; if (!DataTypeUtil.isFixedSizeDataType(child.getDataType())) { size = dataBuffer.array().length; } else if (child.getDataType() == DataTypes.TIMESTAMP) { size = DataTypes.LONG.getSizeInBytes(); } else { size = child.getDataType().getSizeInBytes(); } actualData = getDataObject(dataBuffer, size); return actualData; }
/** * method will read the actual data from the direct dictionary generator * by passing direct dictionary value. * * @param dimColumnEvaluatorInfo * @param dictionaryValue * @return */ private Object getFilterActualValueFromDirectDictionaryValue( DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int dictionaryValue) { if (dimColumnEvaluatorInfo.getDimension().getDataType() == DataTypes.DATE) { return dateDictionaryGenerator.getValueFromSurrogate(dictionaryValue); } else if (dimColumnEvaluatorInfo.getDimension().getDataType() == DataTypes.TIMESTAMP) { return timestampDictionaryGenerator.getValueFromSurrogate(dictionaryValue); } else { throw new RuntimeException("Invalid data type for dierct dictionary"); } }
private void updateNullValue(DataOutputStream dataOutputStream, BadRecordLogHolder logHolder) throws IOException { if (this.carbonDimension.getDataType() == DataTypes.STRING) { dataOutputStream.writeShort(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY.length); dataOutputStream.write(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY); } else { dataOutputStream.writeShort(CarbonCommonConstants.EMPTY_BYTE_ARRAY.length); dataOutputStream.write(CarbonCommonConstants.EMPTY_BYTE_ARRAY); } String message = logHolder.getColumnMessageMap().get(carbonDimension.getColName()); if (null == message) { message = CarbonDataProcessorUtil .prepareFailureReason(carbonDimension.getColName(), carbonDimension.getDataType()); logHolder.getColumnMessageMap().put(carbonDimension.getColName(), message); logHolder.setReason(message); } }
/** * This method will return all the child dimensions under complex dimension */ private static void addAllComplexTypeChildren(CarbonDimension dimension, StringBuilder dimString, String parent) { dimString.append(dimension.getColName()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(dimension.getDataType()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(parent).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(isDictionaryType(dimension)).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(dimension.getColumnId()).append(CarbonCommonConstants.HASH_SPC_CHARACTER); for (int i = 0; i < dimension.getNumberOfChild(); i++) { CarbonDimension childDim = dimension.getListOfChildDimensions().get(i); if (childDim.getNumberOfChild() > 0) { addAllComplexTypeChildren(childDim, dimString, dimension.getColName()); } else { dimString.append(childDim.getColName()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(childDim.getDataType()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(dimension.getColName()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(isDictionaryType(dimension)).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(childDim.getColumnId()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(childDim.getOrdinal()).append(CarbonCommonConstants.HASH_SPC_CHARACTER); } } }
DimensionSpec(ColumnType columnType, CarbonDimension dimension) { super(dimension.getColName(), dimension.getDataType(), columnType); this.inSortColumns = dimension.isSortColumn(); this.doInvertedIndex = dimension.isUseInvertedIndex(); }
/** * create column vector for newly added dimension columns */ private void createVectorForNewlyAddedDimensions() { for (int i = 0; i < queryDimensions.length; i++) { if (!dimensionInfo.getDimensionExists()[i]) { // add a dummy column vector result collector object ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo(); columnVectorInfo.dimension = queryDimensions[i]; if (queryDimensions[i].getDimension().getDataType().equals(DataTypes.TIMESTAMP) || queryDimensions[i].getDimension().getDataType().equals(DataTypes.DATE)) { columnVectorInfo.directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory .getDirectDictionaryGenerator(queryDimensions[i].getDimension().getDataType()); } allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo; } } }
private static void fillParentDetails(Map<Integer, Integer> dimensionToBlockIndexMap, CarbonDimension dimension, Map<Integer, GenericQueryType> complexTypeMap, int[] eachComplexColumnValueSize, Map<String, Dictionary> columnIdToDictionaryMap) { int parentBlockIndex = dimensionToBlockIndexMap.get(dimension.getOrdinal()); GenericQueryType parentQueryType; if (DataTypes.isArrayType(dimension.getDataType())) { parentQueryType = new ArrayQueryType(dimension.getColName(), dimension.getColName(), parentBlockIndex); } else if (DataTypes.isStructType(dimension.getDataType())) { parentQueryType = new StructQueryType(dimension.getColName(), dimension.getColName(), dimensionToBlockIndexMap.get(dimension.getOrdinal())); } else if (DataTypes.isMapType(dimension.getDataType())) { parentQueryType = new MapQueryType(dimension.getColName(), dimension.getColName(), parentBlockIndex); } else { throw new UnsupportedOperationException(dimension.getDataType().getName() + " is not supported"); } complexTypeMap.put(dimension.getOrdinal(), parentQueryType); fillChildrenDetails(eachComplexColumnValueSize, columnIdToDictionaryMap, parentBlockIndex, dimension, parentQueryType); }
/** * This method verifies if the Expression is qualified for Range Expression conversion. * * @param expChild * @return */ private boolean eligibleForRangeExpConv(Expression expChild) { for (Expression exp : expChild.getChildren()) { if (exp instanceof ColumnExpression) { return ((ColumnExpression) exp).isDimension() && ! (((ColumnExpression) exp).getDimension().getDataType().isComplexType()); } } return false; }
protected ColumnFilterInfo getDirectDictionaryValKeyMemberForFilter( ColumnExpression columnExpression, List<String> evaluateResultListFinal, boolean isIncludeFilter, DataType dataType) { List<Integer> surrogates = new ArrayList<Integer>(20); DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory .getDirectDictionaryGenerator(columnExpression.getDimension().getDataType()); // Reading the dictionary value direct getSurrogateValuesForDictionary(evaluateResultListFinal, surrogates, directDictionaryGenerator, dataType); Collections.sort(surrogates); ColumnFilterInfo columnFilterInfo = null; if (surrogates.size() > 0) { columnFilterInfo = new ColumnFilterInfo(); columnFilterInfo.setIncludeFilter(isIncludeFilter); if (!isIncludeFilter) { columnFilterInfo.setExcludeFilterList(surrogates); } else { columnFilterInfo.setFilterList(surrogates); } } return columnFilterInfo; }
private int[] getComplexDimensionParentBlockIndexes(List<ProjectionDimension> queryDimensions) { List<Integer> parentBlockIndexList = new ArrayList<Integer>(); for (ProjectionDimension queryDimension : queryDimensions) { if (queryDimension.getDimension().getDataType().isComplexType()) { if (null != queryDimension.getDimension().getComplexParentDimension()) { if (queryDimension.getDimension().isComplex()) { parentBlockIndexList.add(queryDimension.getDimension().getOrdinal()); } else { parentBlockIndexList.add(queryDimension.getParentDimension().getOrdinal()); } } else { parentBlockIndexList.add(queryDimension.getDimension().getOrdinal()); } } } return ArrayUtils .toPrimitive(parentBlockIndexList.toArray(new Integer[parentBlockIndexList.size()])); }
private boolean isScanRequired(DimensionRawColumnChunk rawColumnChunk, int i) { boolean scanRequired; DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType(); // for no dictionary measure column comparison can be done // on the original data as like measure column if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0) .getDimension().hasEncoding(Encoding.DICTIONARY)) { scanRequired = isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, dataType); } else { scanRequired = isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, rawColumnChunk.getMinMaxFlagArray()[i]); } return scanRequired; }
private boolean isScanRequired(DimensionRawColumnChunk rawColumnChunk, int i) { boolean scanRequired; DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType(); // for no dictionary measure column comparison can be done // on the original data as like measure column if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0) .getDimension().hasEncoding(Encoding.DICTIONARY)) { scanRequired = isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues, dataType); } else { scanRequired = isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues, rawColumnChunk.getMinMaxFlagArray()[i]); } return scanRequired; }
private boolean isScanRequired(DimensionRawColumnChunk rawColumnChunk, int i) { boolean scanRequired; DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType(); // for no dictionary measure column comparison can be done // on the original data as like measure column if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0) .getDimension().hasEncoding(Encoding.DICTIONARY)) { scanRequired = isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, dataType); } else { scanRequired = isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, rawColumnChunk.getMinMaxFlagArray()[i]); } return scanRequired; }
private boolean isScanRequired(DimensionRawColumnChunk rawColumnChunk, int columnIndex) { boolean scanRequired; DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType(); // for no dictionary measure column comparison can be done // on the original data as like measure column if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0).getDimension() .hasEncoding(Encoding.DICTIONARY)) { scanRequired = isScanRequired(rawColumnChunk.getMaxValues()[columnIndex], this.filterRangeValues, dataType); } else { scanRequired = isScanRequired(rawColumnChunk.getMaxValues()[columnIndex], this.filterRangeValues, rawColumnChunk.getMinMaxFlagArray()[columnIndex]); } return scanRequired; }
/** * This method will fill the no dictionary column data * * @param vector * @param columnVectorInfo * @param defaultValue */ private void fillNoDictionaryData(CarbonColumnVector vector, ColumnVectorInfo columnVectorInfo, Object defaultValue) { if (null != defaultValue) { DataType dataType = columnVectorInfo.dimension.getDimension().getDataType(); if (dataType == DataTypes.INT) { vector.putInts(columnVectorInfo.vectorOffset, columnVectorInfo.size, (int) defaultValue); } else if (dataType == DataTypes.LONG || dataType == DataTypes.TIMESTAMP) { vector.putLongs(columnVectorInfo.vectorOffset, columnVectorInfo.size, (long) defaultValue); } else { vector.putBytes(columnVectorInfo.vectorOffset, columnVectorInfo.size, (byte[])defaultValue); } } else { vector.putNulls(columnVectorInfo.vectorOffset, columnVectorInfo.size); } }
/** * This method is used to get default null values for a direct dictionary column * @param currentBlockDimension * @param segmentProperties * @return */ public static byte[] getDefaultNullValue(CarbonDimension currentBlockDimension, SegmentProperties segmentProperties) { byte[] defaultValue = null; DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory .getDirectDictionaryGenerator(currentBlockDimension.getDataType()); int key = directDictionaryGenerator.generateDirectSurrogateKey(null); if (currentBlockDimension.isSortColumn()) { defaultValue = FilterUtil .getMaskKey(key, currentBlockDimension, segmentProperties.getSortColumnsGenerator()); } else { defaultValue = ByteUtil.toXorBytes(key); } return defaultValue; }
private boolean isScanRequired(DimensionRawColumnChunk dimensionRawColumnChunk, int columnIndex) { boolean scanRequired; // for no dictionary measure column comparison can be done // on the original data as like measure column if (DataTypeUtil.isPrimitiveColumn(dimColumnEvaluatorInfo.getDimension().getDataType()) && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)) { scanRequired = isScanRequired(dimensionRawColumnChunk.getMaxValues()[columnIndex], dimensionRawColumnChunk.getMinValues()[columnIndex], dimColumnExecuterInfo.getFilterKeys(), dimColumnEvaluatorInfo.getDimension().getDataType()); } else { scanRequired = isScanRequired(dimensionRawColumnChunk.getMaxValues()[columnIndex], dimensionRawColumnChunk.getMinValues()[columnIndex], dimColumnExecuterInfo.getFilterKeys(), dimensionRawColumnChunk.getMinMaxFlagArray()[columnIndex]); } return scanRequired; }