public ProjectionDimension(CarbonDimension dimension) { super(dimension.getColName()); this.dimension = dimension; }
/** * Get Dimension for columnName from list of dimensions * * @param columnName * @param dimensions * @return */ public static CarbonDimension getCarbonDimension(String columnName, List<CarbonDimension> dimensions) { CarbonDimension carbonDimension = null; for (CarbonDimension dim : dimensions) { if (dim.getColName().equalsIgnoreCase(columnName)) { carbonDimension = dim; break; } } return carbonDimension; }
/** * Find the dimension from metadata by using unique name. As of now we are * taking level name as unique name. But user needs to give one unique name * for each level,that level he needs to mention in query. * * @param dimensions * @param carbonDim * @return */ public static CarbonDimension findDimension(List<CarbonDimension> dimensions, String carbonDim) { CarbonDimension findDim = null; for (CarbonDimension dimension : dimensions) { if (dimension.getColName().equalsIgnoreCase(carbonDim)) { findDim = dimension; break; } } return findDim; }
private boolean checkChildsInSamePath(CarbonDimension parentDimension, CarbonDimension childDimension) { if (parentDimension.getColName().equals(childDimension.getColName())) { return false; } else if (checkForChildColumns(parentDimension, childDimension)) { return true; } else { return false; } }
/** * to get the normal dimension or the primitive dimension of the complex type * * @return primitive dimension of a table */ public CarbonDimension getPrimitiveDimensionByName(String columnName) { List<CarbonDimension> dimList = tablePrimitiveDimensionsMap.get(tableUniqueName); for (CarbonDimension dim : dimList) { if (!dim.isInvisible() && dim.getColName().equalsIgnoreCase(columnName)) { return dim; } } return null; }
@Override public boolean evaluate(Object object) { return ((CarbonColumn) object).getColName().equalsIgnoreCase(dimension.getColName()); } });
/** * Method to get the list of sort columns * * @param tableName * @return List of Sort column */ public List<String> getSortColumns(String tableName) { List<String> sort_columsList = new ArrayList<String>(allDimensions.size()); List<CarbonDimension> carbonDimensions = tableDimensionsMap.get(tableName); for (CarbonDimension dim : carbonDimensions) { if (dim.isSortColumn()) { sort_columsList.add(dim.getColName()); } } return sort_columsList; }
/** * For complex dimensions, check if the dimension already exists in the projection list or not * * @param dimension * @param projectionDimensions * @return */ private boolean isAlreadyExists(CarbonDimension dimension, List<ProjectionDimension> projectionDimensions) { boolean exists = false; for (ProjectionDimension projectionDimension : projectionDimensions) { if (dimension.getColName().equals(projectionDimension.getColumnName())) { exists = true; break; } } return exists; }
private void updateNullValue(DataOutputStream dataOutputStream, BadRecordLogHolder logHolder) throws IOException { if (this.carbonDimension.getDataType() == DataTypes.STRING) { dataOutputStream.writeShort(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY.length); dataOutputStream.write(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY); } else { dataOutputStream.writeShort(CarbonCommonConstants.EMPTY_BYTE_ARRAY.length); dataOutputStream.write(CarbonCommonConstants.EMPTY_BYTE_ARRAY); } String message = logHolder.getColumnMessageMap().get(carbonDimension.getColName()); if (null == message) { message = CarbonDataProcessorUtil .prepareFailureReason(carbonDimension.getColName(), carbonDimension.getDataType()); logHolder.getColumnMessageMap().put(carbonDimension.getColName(), message); logHolder.setReason(message); } }
/** * This method will return all the child dimensions under complex dimension */ private static void addAllComplexTypeChildren(CarbonDimension dimension, StringBuilder dimString, String parent) { dimString.append(dimension.getColName()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(dimension.getDataType()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(parent).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(isDictionaryType(dimension)).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(dimension.getColumnId()).append(CarbonCommonConstants.HASH_SPC_CHARACTER); for (int i = 0; i < dimension.getNumberOfChild(); i++) { CarbonDimension childDim = dimension.getListOfChildDimensions().get(i); if (childDim.getNumberOfChild() > 0) { addAllComplexTypeChildren(childDim, dimString, dimension.getColName()); } else { dimString.append(childDim.getColName()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(childDim.getDataType()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(dimension.getColName()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(isDictionaryType(dimension)).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(childDim.getColumnId()).append(CarbonCommonConstants.COLON_SPC_CHARACTER) .append(childDim.getOrdinal()).append(CarbonCommonConstants.HASH_SPC_CHARACTER); } } }
@Override public Object convert(Object value, BadRecordLogHolder logHolder) throws RuntimeException { try { String parsedValue = null; String dimensionValue = (String) value; if (dimensionValue == null || dimensionValue.equals(nullFormat)) { parsedValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL; } else { parsedValue = DataTypeUtil.parseValue(dimensionValue, carbonDimension); } if (null == parsedValue) { if ((dimensionValue.length() > 0) || (dimensionValue.length() == 0 && isEmptyBadRecord)) { String message = logHolder.getColumnMessageMap().get(carbonDimension.getColName()); if (null == message) { message = CarbonDataProcessorUtil .prepareFailureReason(carbonDimension.getColName(), carbonDimension.getDataType()); logHolder.getColumnMessageMap().put(carbonDimension.getColName(), message); } logHolder.setReason(message); } return CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY; } else { return dictionaryGenerator.getOrGenerateKey(parsedValue); } } catch (DictionaryGenerationException e) { throw new RuntimeException(e); } }
/** * CarbonFactDataHandler constructor */ public CarbonFactDataHandlerColumnar(CarbonFactDataHandlerModel model) { this.model = model; initParameters(model); this.version = CarbonProperties.getInstance().getFormatVersion(); StringBuffer noInvertedIdxCol = new StringBuffer(); for (CarbonDimension cd : model.getSegmentProperties().getDimensions()) { if (!cd.isUseInvertedIndex()) { noInvertedIdxCol.append(cd.getColName()).append(","); } } if (LOGGER.isDebugEnabled()) { LOGGER.debug("Columns considered as NoInverted Index are " + noInvertedIdxCol.toString()); } }
DimensionSpec(ColumnType columnType, CarbonDimension dimension) { super(dimension.getColName(), dimension.getDataType(), columnType); this.inSortColumns = dimension.isSortColumn(); this.doInvertedIndex = dimension.isUseInvertedIndex(); }
private static void fillParentDetails(Map<Integer, Integer> dimensionToBlockIndexMap, CarbonDimension dimension, Map<Integer, GenericQueryType> complexTypeMap, int[] eachComplexColumnValueSize, Map<String, Dictionary> columnIdToDictionaryMap) { int parentBlockIndex = dimensionToBlockIndexMap.get(dimension.getOrdinal()); GenericQueryType parentQueryType; if (DataTypes.isArrayType(dimension.getDataType())) { parentQueryType = new ArrayQueryType(dimension.getColName(), dimension.getColName(), parentBlockIndex); } else if (DataTypes.isStructType(dimension.getDataType())) { parentQueryType = new StructQueryType(dimension.getColName(), dimension.getColName(), dimensionToBlockIndexMap.get(dimension.getOrdinal())); } else if (DataTypes.isMapType(dimension.getDataType())) { parentQueryType = new MapQueryType(dimension.getColName(), dimension.getColName(), parentBlockIndex); } else { throw new UnsupportedOperationException(dimension.getDataType().getName() + " is not supported"); } complexTypeMap.put(dimension.getOrdinal(), parentQueryType); fillChildrenDetails(eachComplexColumnValueSize, columnIdToDictionaryMap, parentBlockIndex, dimension, parentQueryType); }
List<CarbonDimension> dimLists = tableDimensionsMap.get(tableName); for (CarbonDimension dims : dimLists) { if (dims.getColName().equalsIgnoreCase(colSplits[0])) {
/** * This method update the column Name * * @param schema */ public static Set<String> getSchemaColumnNames(CarbonDataLoadSchema schema) { Set<String> columnNames = new HashSet<String>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); String factTableName = schema.getCarbonTable().getTableName(); List<CarbonDimension> dimensions = schema.getCarbonTable().getDimensionByTableName(factTableName); for (CarbonDimension dimension : dimensions) { columnNames.add(dimension.getColName()); } List<CarbonMeasure> measures = schema.getCarbonTable().getMeasureByTableName(factTableName); for (CarbonMeasure msr : measures) { columnNames.add(msr.getColName()); } return columnNames; }
BloomDataMapWriter(String tablePath, String dataMapName, List<CarbonColumn> indexColumns, Segment segment, String shardName, SegmentProperties segmentProperties, int bloomFilterSize, double bloomFilterFpp, boolean compressBloom) throws IOException { super(tablePath, dataMapName, indexColumns, segment, shardName, segmentProperties, bloomFilterSize, bloomFilterFpp, compressBloom); columnarSplitter = segmentProperties.getFixedLengthKeySplitter(); this.indexCol2MdkIdx = new HashMap<>(); int idx = 0; for (final CarbonDimension dimension : segmentProperties.getDimensions()) { if (!dimension.isGlobalDictionaryEncoding() && !dimension.isDirectDictionaryEncoding()) { continue; } boolean isExistInIndex = CollectionUtils.exists(indexColumns, new Predicate() { @Override public boolean evaluate(Object object) { return ((CarbonColumn) object).getColName().equalsIgnoreCase(dimension.getColName()); } }); if (isExistInIndex) { this.indexCol2MdkIdx.put(dimension.getColName(), idx); } idx++; } }
/** * returns level 2 or more child allDimensions * * @param dimName * @param dimensions * @return list of child allDimensions */ public List<CarbonDimension> getChildren(String dimName, List<CarbonDimension> dimensions) { for (CarbonDimension carbonDimension : dimensions) { if (carbonDimension.getColName().equals(dimName)) { return carbonDimension.getListOfChildDimensions(); } else if (null != carbonDimension.getListOfChildDimensions() && carbonDimension.getListOfChildDimensions().size() > 0) { List<CarbonDimension> childDims = getChildren(dimName, carbonDimension.getListOfChildDimensions()); if (childDims != null) { return childDims; } } } return null; }
if (DataTypes.isArrayType(dataType)) { parentQueryType.addChildren( new ArrayQueryType(dimension.getListOfChildDimensions().get(i).getColName(), dimension.getColName(), ++parentBlockIndex)); } else if (DataTypes.isStructType(dataType)) { parentQueryType.addChildren( new StructQueryType(dimension.getListOfChildDimensions().get(i).getColName(), dimension.getColName(), ++parentBlockIndex)); } else if (DataTypes.isMapType(dataType)) { parentQueryType.addChildren( new MapQueryType(dimension.getListOfChildDimensions().get(i).getColName(), dimension.getColName(), ++parentBlockIndex)); } else { boolean isDirectDictionary = CarbonUtil new PrimitiveQueryType(dimension.getListOfChildDimensions().get(i).getColName(), dimension.getColName(), ++parentBlockIndex, dimension.getListOfChildDimensions().get(i).getDataType(), eachComplexColumnValueSize[dimension.getListOfChildDimensions().get(i)
LOGGER.info("\n columnName: " + dimension.getColName() + "\n columnId: " + dimension.getColumnId() + "\n new distinct values count: " + distinctValues.size() +