public TableSpec(CarbonTable carbonTable) { this.carbonTable = carbonTable; List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(carbonTable.getTableName()); List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(carbonTable.getTableName()); // first calculate total number of columnar field considering column group and complex column numSimpleDimensions = 0; for (CarbonDimension dimension : dimensions) { if (!dimension.isComplex()) { numSimpleDimensions++; } } dimensionSpec = new DimensionSpec[dimensions.size()]; measureSpec = new MeasureSpec[measures.size()]; noDictionaryDimensionSpec = new ArrayList<>(); addDimensions(dimensions); addMeasures(measures); }
public CarbonRowDataWriterProcessorStepImpl(CarbonDataLoadConfiguration configuration, AbstractDataLoadProcessorStep child) { super(configuration, child); this.localDictionaryGeneratorMap = CarbonUtil.getLocalDictionaryModel(configuration.getTableSpec().getCarbonTable()); this.carbonFactHandlers = new CopyOnWriteArrayList<>(); }
ColumnPage page = null; TableSpec spec = model.getTableSpec(); int numDimensions = spec.getNumDimensions(); for (int i = 0; i < numDimensions; i++) { ColumnType type = spec.getDimensionSpec(i).getColumnType(); if ((type == ColumnType.GLOBAL_DICTIONARY) || (type == ColumnType.DIRECT_DICTIONARY)) { page = dictDimensionPages[++dictDimensionIndex]; String fieldName = spec.getDimensionSpec(i).getFieldName(); if (fieldName.equalsIgnoreCase(columnName)) { return page; int numMeasures = spec.getNumMeasures(); for (int i = 0; i < numMeasures; i++) { String fieldName = spec.getMeasureSpec(i).getFieldName(); if (fieldName.equalsIgnoreCase(columnName)) { return measurePages[i];
int noDictIndex = 0; int complexDimIndex = 0; int numDimensions = tableSpec.getNumDimensions(); for (int i = 0; i < numDimensions; i++) { ColumnPageEncoder columnPageEncoder; EncodedColumnPage encodedPage; TableSpec.DimensionSpec spec = tableSpec.getDimensionSpec(i); switch (spec.getColumnType()) { case GLOBAL_DICTIONARY:
int tmpNumNoDictDimIdx = 0; for (int i = 0; i < dictDimensionPages.length + noDictDimensionPages.length; i++) { TableSpec.DimensionSpec spec = tableSpec.getDimensionSpec(i); ColumnType columnType = tableSpec.getDimensionSpec(i).getColumnType(); ColumnPage page; if (ColumnType.GLOBAL_DICTIONARY == columnType for (int i = 0; i < measurePages.length; i++) { ColumnPageEncoderMeta columnPageEncoderMeta = new ColumnPageEncoderMeta( model.getTableSpec().getMeasureSpec(i), dataTypes[i], columnCompressor); ColumnPage page; if (DataTypes.isDecimal(columnPageEncoderMeta.getSchemaDataType())) {
carbonFactDataHandlerModel.setColumnCompressor(loadModel.getColumnCompressor()); carbonFactDataHandlerModel.tableSpec = new TableSpec(carbonTable); DataMapWriterListener listener = new DataMapWriterListener(); listener.registerAllWriter(
private EncodedColumnPage[] encodeAndCompressMeasures() throws MemoryException, IOException { EncodedColumnPage[] encodedMeasures = new EncodedColumnPage[measurePages.length]; for (int i = 0; i < measurePages.length; i++) { ColumnPageEncoder encoder = encodingFactory.createEncoder( model.getTableSpec().getMeasureSpec(i), measurePages[i]); encodedMeasures[i] = encoder.encode(measurePages[i]); } return encodedMeasures; }
TableSpec tableSpec = model.getTableSpec(); List<TableSpec.DimensionSpec> noDictionaryDimensionSpec = tableSpec.getNoDictionaryDimensionSpec(); Object[] noDictAndComplex = WriteStepRowUtil.getNoDictAndComplexDimension(row); for (int i = 0; i < noDictAndComplex.length; i++) {
configuration.setWritingCoresCount((short) 1); TableSpec tableSpec = new TableSpec(carbonTable); configuration.setTableSpec(tableSpec); if (loadModel.getSdkWriterCores() > 0) {
public DataWriterProcessorStepImpl(CarbonDataLoadConfiguration configuration, AbstractDataLoadProcessorStep child) { super(configuration, child); this.localDictionaryGeneratorMap = CarbonUtil.getLocalDictionaryModel(configuration.getTableSpec().getCarbonTable()); }
public DataWriterProcessorStepImpl(CarbonDataLoadConfiguration configuration) { super(configuration, null); this.localDictionaryGeneratorMap = CarbonUtil.getLocalDictionaryModel(configuration.getTableSpec().getCarbonTable()); }
public DataWriterBatchProcessorStepImpl(CarbonDataLoadConfiguration configuration, AbstractDataLoadProcessorStep child) { super(configuration, child); this.localDictionaryGeneratorMap = CarbonUtil.getLocalDictionaryModel(configuration.getTableSpec().getCarbonTable()); }
private String[] getStoreLocation() { return CarbonDataProcessorUtil .getLocalDataFolderLocation(configuration.getTableSpec().getCarbonTable(), String.valueOf(configuration.getTaskNo()), configuration.getSegmentId(), false, false); }
private String[] getStoreLocation() { String[] storeLocation = CarbonDataProcessorUtil .getLocalDataFolderLocation(this.configuration.getTableSpec().getCarbonTable(), String.valueOf(configuration.getTaskNo()), configuration.getSegmentId(), false, false); CarbonDataProcessorUtil.createLocations(storeLocation); return storeLocation; }
private String[] getStoreLocation() { String[] storeLocation = CarbonDataProcessorUtil .getLocalDataFolderLocation(configuration.getTableSpec().getCarbonTable(), String.valueOf(configuration.getTaskNo()), configuration.getSegmentId(), false, false); CarbonDataProcessorUtil.createLocations(storeLocation); return storeLocation; }
public CarbonFactDataWriterImplV3(CarbonFactDataHandlerModel model) { super(model); String blockletSize = model.getTableSpec().getCarbonTable().getTableInfo().getFactTable().getTableProperties() .get(TABLE_BLOCKLET_SIZE); if (blockletSize == null) { blockletSize = CarbonProperties.getInstance().getProperty( BLOCKLET_SIZE_IN_MB, BLOCKLET_SIZE_IN_MB_DEFAULT_VALUE); } blockletSizeThreshold = Long.parseLong(blockletSize) << 20; if (blockletSizeThreshold > fileSizeInBytes) { blockletSizeThreshold = fileSizeInBytes; LOGGER.info("Blocklet size configure for table is: " + blockletSizeThreshold); } blockletDataHolder = new BlockletDataHolder(fallbackExecutorService, model); isSorted = model.getSortScope() != NO_SORT; }
new RawRowComparator(sortColumnRangeInfo.getSortColumnIndex(), sortColumnRangeInfo.getIsSortColumnNoDict(), CarbonDataProcessorUtil .getNoDictDataTypes(configuration.getTableSpec().getCarbonTable()))); new RawRowComparator(sortColumnRangeInfo.getSortColumnIndex(), sortColumnRangeInfo.getIsSortColumnNoDict(), CarbonDataProcessorUtil .getNoDictDataTypes(configuration.getTableSpec().getCarbonTable())));
configuration.getTableIdentifier().getCarbonTableIdentifier(); CarbonProperties carbonProperties = CarbonProperties.getInstance(); parameters.setCarbonTable(configuration.getTableSpec().getCarbonTable()); parameters.setDatabaseName(tableIdentifier.getDatabaseName()); parameters.setTableName(tableIdentifier.getTableName()); parameters.setMeasureDataType(measureDataType); parameters.setNoDictDataType(CarbonDataProcessorUtil .getNoDictDataTypes(configuration.getTableSpec().getCarbonTable())); Map<String, DataType[]> noDictSortAndNoSortDataTypes = CarbonDataProcessorUtil .getNoDictSortAndNoSortDataTypes(configuration.getTableSpec().getCarbonTable()); parameters.setNoDictSortDataType(noDictSortAndNoSortDataTypes.get("noDictSortDataTypes")); parameters.setNoDictNoSortDataType(noDictSortAndNoSortDataTypes.get("noDictNoSortDataTypes"));
CarbonTable carbonTable = configuration.getTableSpec().getCarbonTable(); listener = new DataMapWriterListener(); listener.registerAllWriter( configuration.getTableSpec().getCarbonTable(), configuration.getSegmentId(), CarbonTablePath.getShardName(