EncodedColumnPage encodedPage; TableSpec.DimensionSpec spec = tableSpec.getDimensionSpec(i); switch (spec.getColumnType()) { case GLOBAL_DICTIONARY: case DIRECT_DICTIONARY: default: throw new IllegalArgumentException("unsupported dimension type:" + spec .getColumnType());
private void addDimensions(List<CarbonDimension> dimensions) { int dimIndex = 0; for (int i = 0; i < dimensions.size(); i++) { CarbonDimension dimension = dimensions.get(i); if (dimension.isComplex()) { DimensionSpec spec = new DimensionSpec(ColumnType.COMPLEX, dimension); dimensionSpec[dimIndex++] = spec; noDictionaryDimensionSpec.add(spec); } else if (dimension.getDataType() == DataTypes.TIMESTAMP && !dimension .isDirectDictionaryEncoding()) { DimensionSpec spec = new DimensionSpec(ColumnType.PLAIN_VALUE, dimension); dimensionSpec[dimIndex++] = spec; noDictionaryDimensionSpec.add(spec); } else if (dimension.isDirectDictionaryEncoding()) { DimensionSpec spec = new DimensionSpec(ColumnType.DIRECT_DICTIONARY, dimension); dimensionSpec[dimIndex++] = spec; } else if (dimension.isGlobalDictionaryEncoding()) { DimensionSpec spec = new DimensionSpec(ColumnType.GLOBAL_DICTIONARY, dimension); dimensionSpec[dimIndex++] = spec; } else { DimensionSpec spec = new DimensionSpec(ColumnType.PLAIN_VALUE, dimension); dimensionSpec[dimIndex++] = spec; noDictionaryDimensionSpec.add(spec); } } }
int numDimensions = spec.getNumDimensions(); for (int i = 0; i < numDimensions; i++) { ColumnType type = spec.getDimensionSpec(i).getColumnType(); if ((type == ColumnType.GLOBAL_DICTIONARY) || (type == ColumnType.DIRECT_DICTIONARY)) { page = dictDimensionPages[++dictDimensionIndex]; String fieldName = spec.getDimensionSpec(i).getFieldName(); if (fieldName.equalsIgnoreCase(columnName)) { return page;
for (int i = 0; i < dictDimensionPages.length + noDictDimensionPages.length; i++) { TableSpec.DimensionSpec spec = tableSpec.getDimensionSpec(i); ColumnType columnType = tableSpec.getDimensionSpec(i).getColumnType(); ColumnPage page; if (ColumnType.GLOBAL_DICTIONARY == columnType model.getColumnLocalDictGenMap().get(spec.getFieldName()); DataType dataType = DataTypes.STRING; if (DataTypes.VARCHAR == spec.getSchemaDataType()) { dataType = DataTypes.VARCHAR; columnPageEncoderMeta, pageSize, localDictionaryGenerator, false); } else { if (DataTypeUtil.isPrimitiveColumn(spec.getSchemaDataType())) { if (spec.getSchemaDataType() == DataTypes.TIMESTAMP) { columnPageEncoderMeta = new ColumnPageEncoderMeta(spec, DataTypes.LONG, columnCompressor); } else { columnPageEncoderMeta = new ColumnPageEncoderMeta(spec, spec.getSchemaDataType(), columnCompressor); if (DataTypes.isDecimal(spec.getSchemaDataType())) { page = ColumnPage.newDecimalPage(columnPageEncoderMeta, pageSize); } else { } else if (DataTypeUtil.isPrimitiveColumn(spec.getSchemaDataType())) { if (spec.getSchemaDataType() == DataTypes.TIMESTAMP) { page.setStatsCollector(PrimitivePageStatsCollector.newInstance(DataTypes.LONG)); } else {
Object[] noDictAndComplex = WriteStepRowUtil.getNoDictAndComplexDimension(row); for (int i = 0; i < noDictAndComplex.length; i++) { if (noDictionaryDimensionSpec.get(i).getSchemaDataType() == DataTypes.VARCHAR) { byte[] valueWithLength = addIntLengthToByteArray((byte[]) noDictAndComplex[i]);
private ColumnPageEncoder createEncoderForDimensionLegacy(TableSpec.DimensionSpec dimensionSpec) { switch (dimensionSpec.getColumnType()) { case GLOBAL_DICTIONARY: return new DictDimensionIndexCodec( dimensionSpec.isInSortColumns(), dimensionSpec.isInSortColumns() && dimensionSpec.isDoInvertedIndex()) .createEncoder(null); case DIRECT_DICTIONARY: return new DirectDictDimensionIndexCodec( dimensionSpec.isInSortColumns(), dimensionSpec.isInSortColumns() && dimensionSpec.isDoInvertedIndex()) .createEncoder(null); case PLAIN_VALUE: return new HighCardDictDimensionIndexCodec( dimensionSpec.isInSortColumns(), dimensionSpec.isInSortColumns() && dimensionSpec.isDoInvertedIndex(), dimensionSpec.getSchemaDataType() == DataTypes.VARCHAR) .createEncoder(null); default: throw new RuntimeException("unsupported dimension type: " + dimensionSpec.getColumnType()); } }
/** * No dictionary and complex dimensions of the table * * @return */ public DimensionSpec[] getNoDictAndComplexDimensions() { List<DimensionSpec> noDictAndComplexDimensions = new ArrayList<>(); for (int i = 0; i < dimensionSpec.length; i++) { if (dimensionSpec[i].getColumnType() == ColumnType.PLAIN_VALUE || dimensionSpec[i].getColumnType() == ColumnType.COMPLEX_PRIMITIVE || dimensionSpec[i].getColumnType() == ColumnType.COMPLEX) { noDictAndComplexDimensions.add(dimensionSpec[i]); } } return noDictAndComplexDimensions.toArray(new DimensionSpec[noDictAndComplexDimensions.size()]); }
/** * Check whether the column is sort column and inverted index column * * @param isComplexPrimitive * @param columnSpec * @return */ private static boolean isInvertedIndex(boolean isComplexPrimitive, TableSpec.ColumnSpec columnSpec) { boolean isSort; boolean isInvertedIndex = false; if (columnSpec instanceof TableSpec.DimensionSpec && !isComplexPrimitive) { isSort = ((TableSpec.DimensionSpec) columnSpec).isInSortColumns(); isInvertedIndex = isSort && ((TableSpec.DimensionSpec) columnSpec).isDoInvertedIndex(); } return isInvertedIndex; }
private ColumnPageEncoder createEncoderForDimension(TableSpec.DimensionSpec columnSpec, ColumnPage inputPage) { switch (columnSpec.getColumnType()) { case GLOBAL_DICTIONARY: case DIRECT_DICTIONARY: case PLAIN_VALUE: return new DirectCompressCodec(inputPage.getDataType()).createEncoder(null); case COMPLEX: return new ComplexDimensionIndexCodec(false, false).createEncoder(null); default: throw new RuntimeException("unsupported dimension type: " + columnSpec.getColumnType()); } }