public CarbonRowDataWriterProcessorStepImpl(CarbonDataLoadConfiguration configuration, AbstractDataLoadProcessorStep child) { super(configuration, child); this.localDictionaryGeneratorMap = CarbonUtil.getLocalDictionaryModel(configuration.getTableSpec().getCarbonTable()); this.carbonFactHandlers = new CopyOnWriteArrayList<>(); }
public DataWriterProcessorStepImpl(CarbonDataLoadConfiguration configuration, AbstractDataLoadProcessorStep child) { super(configuration, child); this.localDictionaryGeneratorMap = CarbonUtil.getLocalDictionaryModel(configuration.getTableSpec().getCarbonTable()); }
public DataWriterProcessorStepImpl(CarbonDataLoadConfiguration configuration) { super(configuration, null); this.localDictionaryGeneratorMap = CarbonUtil.getLocalDictionaryModel(configuration.getTableSpec().getCarbonTable()); }
public DataWriterBatchProcessorStepImpl(CarbonDataLoadConfiguration configuration, AbstractDataLoadProcessorStep child) { super(configuration, child); this.localDictionaryGeneratorMap = CarbonUtil.getLocalDictionaryModel(configuration.getTableSpec().getCarbonTable()); }
private String[] getStoreLocation() { return CarbonDataProcessorUtil .getLocalDataFolderLocation(configuration.getTableSpec().getCarbonTable(), String.valueOf(configuration.getTaskNo()), configuration.getSegmentId(), false, false); }
private String[] getStoreLocation() { String[] storeLocation = CarbonDataProcessorUtil .getLocalDataFolderLocation(this.configuration.getTableSpec().getCarbonTable(), String.valueOf(configuration.getTaskNo()), configuration.getSegmentId(), false, false); CarbonDataProcessorUtil.createLocations(storeLocation); return storeLocation; }
private String[] getStoreLocation() { String[] storeLocation = CarbonDataProcessorUtil .getLocalDataFolderLocation(configuration.getTableSpec().getCarbonTable(), String.valueOf(configuration.getTaskNo()), configuration.getSegmentId(), false, false); CarbonDataProcessorUtil.createLocations(storeLocation); return storeLocation; }
public CarbonFactDataWriterImplV3(CarbonFactDataHandlerModel model) { super(model); String blockletSize = model.getTableSpec().getCarbonTable().getTableInfo().getFactTable().getTableProperties() .get(TABLE_BLOCKLET_SIZE); if (blockletSize == null) { blockletSize = CarbonProperties.getInstance().getProperty( BLOCKLET_SIZE_IN_MB, BLOCKLET_SIZE_IN_MB_DEFAULT_VALUE); } blockletSizeThreshold = Long.parseLong(blockletSize) << 20; if (blockletSizeThreshold > fileSizeInBytes) { blockletSizeThreshold = fileSizeInBytes; LOGGER.info("Blocklet size configure for table is: " + blockletSizeThreshold); } blockletDataHolder = new BlockletDataHolder(fallbackExecutorService, model); isSorted = model.getSortScope() != NO_SORT; }
new RawRowComparator(sortColumnRangeInfo.getSortColumnIndex(), sortColumnRangeInfo.getIsSortColumnNoDict(), CarbonDataProcessorUtil .getNoDictDataTypes(configuration.getTableSpec().getCarbonTable()))); new RawRowComparator(sortColumnRangeInfo.getSortColumnIndex(), sortColumnRangeInfo.getIsSortColumnNoDict(), CarbonDataProcessorUtil .getNoDictDataTypes(configuration.getTableSpec().getCarbonTable())));
configuration.getTableIdentifier().getCarbonTableIdentifier(); CarbonProperties carbonProperties = CarbonProperties.getInstance(); parameters.setCarbonTable(configuration.getTableSpec().getCarbonTable()); parameters.setDatabaseName(tableIdentifier.getDatabaseName()); parameters.setTableName(tableIdentifier.getTableName()); parameters.setMeasureDataType(measureDataType); parameters.setNoDictDataType(CarbonDataProcessorUtil .getNoDictDataTypes(configuration.getTableSpec().getCarbonTable())); Map<String, DataType[]> noDictSortAndNoSortDataTypes = CarbonDataProcessorUtil .getNoDictSortAndNoSortDataTypes(configuration.getTableSpec().getCarbonTable()); parameters.setNoDictSortDataType(noDictSortAndNoSortDataTypes.get("noDictSortDataTypes")); parameters.setNoDictNoSortDataType(noDictSortAndNoSortDataTypes.get("noDictNoSortDataTypes"));
CarbonTable carbonTable = configuration.getTableSpec().getCarbonTable(); listener = new DataMapWriterListener(); listener.registerAllWriter( configuration.getTableSpec().getCarbonTable(), configuration.getSegmentId(), CarbonTablePath.getShardName(