/** * This method will return all the child dimensions under complex dimension */ private static void addAllComplexTypeChildren(CarbonDimension dimension, StringBuilder dimString, String parent) { dimString.append(dimension.getColName()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(dimension.getDataType()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(parent).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(isDictionaryType(dimension)).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(dimension.getColumnId()).append(CarbonCommonConstants.HASH_SPC_CHARACTER); for (int i = 0; i < dimension.getNumberOfChild(); i++) { CarbonDimension childDim = dimension.getListOfChildDimensions().get(i); if (childDim.getNumberOfChild() > 0) { addAllComplexTypeChildren(childDim, dimString, dimension.getColName()); } else { dimString.append(childDim.getColName()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(childDim.getDataType()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(dimension.getColName()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(isDictionaryType(dimension)).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(childDim.getColumnId()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(childDim.getOrdinal()).append(CarbonCommonConstants.HASH_SPC_CHARACTER); } } }
public static void getComplexNoDictionaryMapping(List<CarbonDimension> carbonDimensions, List<Integer> complexNoDictionary) { for (CarbonDimension carbonDimension : carbonDimensions) { if (carbonDimension.isComplex()) { getComplexNoDictionaryMapping(carbonDimension.getListOfChildDimensions(), complexNoDictionary); } else { // This is primitive type. Check the encoding for NoDictionary. if (!carbonDimension.hasEncoding(Encoding.DICTIONARY)) { complexNoDictionary.add(carbonDimension.getOrdinal()); } } } }
private static void fillCollumnSchemaListForComplexDims( List<CarbonDimension> carbonDimensionsList, List<ColumnSchema> wrapperColumnSchemaList) { for (CarbonDimension carbonDimension : carbonDimensionsList) { wrapperColumnSchemaList.add(carbonDimension.getColumnSchema()); List<CarbonDimension> childDims = carbonDimension.getListOfChildDimensions(); if (null != childDims && childDims.size() > 0) { fillCollumnSchemaListForComplexDims(childDims, wrapperColumnSchemaList); } } }
DimensionSpec(ColumnType columnType, CarbonDimension dimension) { super(dimension.getColName(), dimension.getDataType(), columnType); this.inSortColumns = dimension.isSortColumn(); this.doInvertedIndex = dimension.isUseInvertedIndex(); }
private boolean checkForChildColumns(CarbonDimension parentDimension, CarbonDimension childDimension) { boolean output = false; if (parentDimension.getOrdinal() == childDimension.getOrdinal()) { output = true; } else if (parentDimension.getNumberOfChild() > 0) { for (int i = 0; i < parentDimension.getNumberOfChild() && !output; i++) { output = checkForChildColumns(parentDimension.getListOfChildDimensions().get(i), childDimension); } } else { output = false; } return output; }
/** * to get the normal dimension or the primitive dimension of the complex type * * @return primitive dimension of a table */ public CarbonDimension getPrimitiveDimensionByName(String columnName) { List<CarbonDimension> dimList = tablePrimitiveDimensionsMap.get(tableUniqueName); for (CarbonDimension dim : dimList) { if (!dim.isInvisible() && dim.getColName().equalsIgnoreCase(columnName)) { return dim; } } return null; }
private static int fillChildrenDetails(int[] eachComplexColumnValueSize, Map<String, Dictionary> columnIdToDictionaryMap, int parentBlockIndex, CarbonDimension dimension, GenericQueryType parentQueryType) { for (int i = 0; i < dimension.getNumberOfChild(); i++) { DataType dataType = dimension.getListOfChildDimensions().get(i).getDataType(); if (DataTypes.isArrayType(dataType)) { parentQueryType.addChildren( new ArrayQueryType(dimension.getListOfChildDimensions().get(i).getColName(), dimension.getColName(), ++parentBlockIndex)); } else if (DataTypes.isStructType(dataType)) { parentQueryType.addChildren( new StructQueryType(dimension.getListOfChildDimensions().get(i).getColName(), dimension.getColName(), ++parentBlockIndex)); } else if (DataTypes.isMapType(dataType)) { parentQueryType.addChildren( new MapQueryType(dimension.getListOfChildDimensions().get(i).getColName(), dimension.getColName(), ++parentBlockIndex)); } else { boolean isDirectDictionary = CarbonUtil .hasEncoding(dimension.getListOfChildDimensions().get(i).getEncoder(), Encoding.DIRECT_DICTIONARY); boolean isDictionary = CarbonUtil .hasEncoding(dimension.getListOfChildDimensions().get(i).getEncoder(), Encoding.DICTIONARY); new PrimitiveQueryType(dimension.getListOfChildDimensions().get(i).getColName(), dimension.getColName(), ++parentBlockIndex, dimension.getListOfChildDimensions().get(i).getDataType(), eachComplexColumnValueSize[dimension.getListOfChildDimensions().get(i) .getComplexTypeOrdinal()], columnIdToDictionaryMap
private static void fillChildren(CarbonTable carbontable, GenericQueryType parentQueryType, CarbonDimension dimension, int parentBlockIndex, Cache<DictionaryColumnUniqueIdentifier, Dictionary> cache) throws IOException { for (int i = 0; i < dimension.getNumberOfChild(); i++) { CarbonDimension child = dimension.getListOfChildDimensions().get(i); DataType dataType = child.getDataType(); GenericQueryType queryType = null; if (DataTypes.isArrayType(dataType)) { queryType = new ArrayQueryType(child.getColName(), dimension.getColName(), ++parentBlockIndex); new StructQueryType(child.getColName(), dimension.getColName(), ++parentBlockIndex); parentQueryType.addChildren(queryType); } else { boolean isDirectDictionary = CarbonUtil.hasEncoding(child.getEncoder(), Encoding.DIRECT_DICTIONARY); boolean isDictionary = CarbonUtil.hasEncoding(child.getEncoder(), Encoding.DICTIONARY); Dictionary dictionary = null; if (isDictionary) { DictionaryColumnUniqueIdentifier dictionarIdentifier = new DictionaryColumnUniqueIdentifier(carbontable.getAbsoluteTableIdentifier(), child.getColumnIdentifier(), child.getDataType(), dictionaryPath); dictionary = cache.get(dictionarIdentifier); new PrimitiveQueryType(child.getColName(), dimension.getColName(), ++parentBlockIndex, child.getDataType(), 4, dictionary, isDirectDictionary);
int noDicAndComp = 0; for (CarbonDimension dim : allDimensions) { if (!dim.isComplex() && !dim.hasEncoding(Encoding.DICTIONARY) && dim.getDataType() == DataTypes.VARCHAR) { varcharDimIdxInNoDict.add(dim.getOrdinal() - dictDimCount); if (!dim.hasEncoding(Encoding.DICTIONARY)) { noDicAndComplexColumns[noDicAndComp++] = new CarbonColumn(dim.getColumnSchema(), dim.getOrdinal(), dim.getSchemaOrdinal());
private boolean isScanRequired(DimensionRawColumnChunk rawColumnChunk, int i) { boolean scanRequired; DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType(); // for no dictionary measure column comparison can be done // on the original data as like measure column if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0) .getDimension().hasEncoding(Encoding.DICTIONARY)) { scanRequired = isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, dataType); } else { scanRequired = isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, rawColumnChunk.getMinMaxFlagArray()[i]); } return scanRequired; }
CarbonDimension carbonDimension) throws IOException { String dictionaryPath = null; ColumnIdentifier columnIdentifier = carbonDimension.getColumnIdentifier(); String dicPath = dictionarySourceAbsoluteTableIdentifier.getDictionaryPath(); if (null != dicPath && !dicPath.trim().isEmpty()) { dictionaryPath = dicPath; if (null != carbonDimension.getColumnSchema().getParentColumnTableRelations() && carbonDimension.getColumnSchema().getParentColumnTableRelations().size() == 1) { dictionarySourceAbsoluteTableIdentifier = QueryUtil.getTableIdentifierForColumn(carbonDimension); columnIdentifier = new ColumnIdentifier( carbonDimension.getColumnSchema().getParentColumnTableRelations().get(0).getColumnId(), carbonDimension.getColumnProperties(), carbonDimension.getDataType()); columnIdentifier, carbonDimension.getDataType(), dictionaryPath); CacheProvider cacheProvider = CacheProvider.getInstance(); Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache =
carbonDimension.getColumnIdentifier(), carbonDimension.getDataType()); try { if (carbonDimension.hasEncoding(Encoding.DIRECT_DICTIONARY) || carbonColumn.getDataType() == DataTypes.DATE) { dictionaryGenerator = new DirectDictionary(DirectDictionaryKeyGeneratorFactory .getDirectDictionaryGenerator(carbonDimension.getDataType(), getDateFormat(carbonDimension))); isDirectDictionary = true; } else if (carbonDimension.hasEncoding(Encoding.DICTIONARY)) { CacheProvider cacheProvider = CacheProvider.getInstance(); Cache<DictionaryColumnUniqueIdentifier, Dictionary> cache = dictionaryMessage.setColumnName(carbonDimension.getColName());
for (int i = 0; i < queryDimension.size(); i++) { ProjectionDimension dim = queryDimension.get(i); if (dim.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) { DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory .getDirectDictionaryGenerator(dim.getDimension().getDataType()); fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), CarbonSparkDataSourceUtil.convertCarbonToSparkDataType(generator.getReturnType()), true, null); } else if (!dim.getDimension().hasEncoding(Encoding.DICTIONARY)) { if (dim.getDimension().getDataType() == DataTypes.STRING || dim.getDimension().getDataType() == DataTypes.VARCHAR || dim.getDimension() .getColumnSchema().isLocalDictColumn()) { this.isNoDictStringField[dim.getOrdinal()] = true; CarbonSparkDataSourceUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true, null); } else if (dim.getDimension().isComplex()) { fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), CarbonSparkDataSourceUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true, null); } else {
private void addDimensions(List<CarbonDimension> dimensions) { int dimIndex = 0; for (int i = 0; i < dimensions.size(); i++) { CarbonDimension dimension = dimensions.get(i); if (dimension.isComplex()) { DimensionSpec spec = new DimensionSpec(ColumnType.COMPLEX, dimension); dimensionSpec[dimIndex++] = spec; noDictionaryDimensionSpec.add(spec); } else if (dimension.getDataType() == DataTypes.TIMESTAMP && !dimension .isDirectDictionaryEncoding()) { DimensionSpec spec = new DimensionSpec(ColumnType.PLAIN_VALUE, dimension); dimensionSpec[dimIndex++] = spec; noDictionaryDimensionSpec.add(spec); } else if (dimension.isDirectDictionaryEncoding()) { DimensionSpec spec = new DimensionSpec(ColumnType.DIRECT_DICTIONARY, dimension); dimensionSpec[dimIndex++] = spec; } else if (dimension.isGlobalDictionaryEncoding()) { DimensionSpec spec = new DimensionSpec(ColumnType.GLOBAL_DICTIONARY, dimension); dimensionSpec[dimIndex++] = spec; } else { DimensionSpec spec = new DimensionSpec(ColumnType.PLAIN_VALUE, dimension); dimensionSpec[dimIndex++] = spec; noDictionaryDimensionSpec.add(spec); } } }
if (queryDimensions[i].getDimension().hasEncoding(Encoding.IMPLICIT)) { ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo(); implictColumnList.add(columnVectorInfo); columnVectorInfo.dimension = queryDimensions[i]; columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal(); allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo; } else if (!queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY)) { ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo(); noDictInfoList.add(columnVectorInfo); columnVectorInfo.dimension = queryDimensions[i]; columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal(); allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo; } else if (queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) { ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo(); dictInfoList.add(columnVectorInfo); columnVectorInfo.dimension = queryDimensions[i]; columnVectorInfo.directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory .getDirectDictionaryGenerator(queryDimensions[i].getDimension().getDataType()); columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal(); allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo; } else if (queryDimensions[i].getDimension().isComplex()) { ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo(); complexList.add(columnVectorInfo); columnVectorInfo.dimension = queryDimensions[i]; columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal(); columnVectorInfo.genericQueryType = executionInfo.getComlexDimensionInfoMap().get(columnVectorInfo.ordinal); dictInfoList.add(columnVectorInfo);
if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) { presentDimension.add(queryDimension); isDimensionExists[dimIndex] = true; dimensionInfo.dataType[queryDimension.getOrdinal()] = queryDimension.getDimension().getDataType(); } else { for (CarbonDimension tableDimension : tableBlockDimensions) { tableDimension)) { ProjectionDimension currentBlockDimension = new ProjectionDimension(tableDimension); tableDimension.getColumnSchema() .setPrecision(queryDimension.getDimension().getColumnSchema().getPrecision()); tableDimension.getColumnSchema() .setScale(queryDimension.getDimension().getColumnSchema().getScale()); tableDimension.getColumnSchema() .setDefaultValue(queryDimension.getDimension().getDefaultValue()); currentBlockDimension.setOrdinal(queryDimension.getOrdinal()); presentDimension.add(currentBlockDimension); isDimensionExists[dimIndex] = true; dimensionInfo.dataType[currentBlockDimension.getOrdinal()] = currentBlockDimension.getDimension().getDataType(); break; if (null != queryDimension.getDimension().getComplexParentDimension()) { currentBlockDimension = new ProjectionDimension(queryDimension.getDimension()); } else { isDimensionExists[dimIndex] = true; dimensionInfo.dataType[currentBlockDimension.getOrdinal()] =
/** * Below method will be used to fill the children dimension column id * * @param queryDimensions query dimension * @param dictionaryDimensionFromQuery dictionary dimension for query */ private static void getChildDimensionDictionaryDetail(CarbonDimension queryDimensions, Set<String> dictionaryDimensionFromQuery) { for (int j = 0; j < queryDimensions.getNumberOfChild(); j++) { List<Encoding> encodingList = queryDimensions.getListOfChildDimensions().get(j).getEncoder(); if (queryDimensions.getListOfChildDimensions().get(j).getNumberOfChild() > 0) { getChildDimensionDictionaryDetail(queryDimensions.getListOfChildDimensions().get(j), dictionaryDimensionFromQuery); } else if (CarbonUtil.hasEncoding(encodingList, Encoding.DICTIONARY) && !CarbonUtil .hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) { dictionaryDimensionFromQuery .add(queryDimensions.getListOfChildDimensions().get(j).getColumnId()); } } }
for (int i = 0; i < dimensions.size(); i++) { CarbonDimension dims = dimensions.get(i); if (dims.hasEncoding(Encoding.DICTIONARY)) { if (DataTypeUtil.isPrimitiveColumn(dims.getDataType()) && !dims.isComplex()) { .getDataBasedOnDataTypeForNoDictionaryColumn((byte[]) row[i], dims.getDataType()); if (null != preparedRow[i] && dims.getDataType() == DataTypes.TIMESTAMP) { preparedRow[i] = (long) preparedRow[i] / 1000L;
/** * Below method will be used to fill the children dimension column id */ private static void getChildDimensionOrdinal(CarbonDimension queryDimensions, Set<Integer> filterDimensionsOrdinal) { for (int j = 0; j < queryDimensions.getNumberOfChild(); j++) { List<Encoding> encodingList = queryDimensions.getListOfChildDimensions().get(j).getEncoder(); if (queryDimensions.getListOfChildDimensions().get(j).getNumberOfChild() > 0) { getChildDimensionOrdinal(queryDimensions.getListOfChildDimensions().get(j), filterDimensionsOrdinal); } else if (!CarbonUtil.hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) { filterDimensionsOrdinal.add(queryDimensions.getListOfChildDimensions().get(j).getOrdinal()); } } }
private int[] getComplexDimensionParentBlockIndexes(List<ProjectionDimension> queryDimensions) { List<Integer> parentBlockIndexList = new ArrayList<Integer>(); for (ProjectionDimension queryDimension : queryDimensions) { if (queryDimension.getDimension().getDataType().isComplexType()) { if (null != queryDimension.getDimension().getComplexParentDimension()) { if (queryDimension.getDimension().isComplex()) { parentBlockIndexList.add(queryDimension.getDimension().getOrdinal()); } else { parentBlockIndexList.add(queryDimension.getParentDimension().getOrdinal()); } } else { parentBlockIndexList.add(queryDimension.getDimension().getOrdinal()); } } } return ArrayUtils .toPrimitive(parentBlockIndexList.toArray(new Integer[parentBlockIndexList.size()])); }