public SortIntermediateFileMerger(SortParameters parameters) { this.parameters = parameters; // processed file list this.procFiles = new ArrayList<File>(CarbonCommonConstants.CONSTANT_SIZE_TEN); this.executorService = Executors.newFixedThreadPool(parameters.getNumberOfCores(), new CarbonThreadFactory("SafeIntermediateMergerPool:" + parameters.getTableName())); mergerTask = new ArrayList<>(); }
private void initParameters(CarbonFactDataHandlerModel model) { this.numberOfCores = model.getNumberOfCores(); blockletProcessingCount = new AtomicInteger(0); producerExecutorService = Executors.newFixedThreadPool(model.getNumberOfCores(), new CarbonThreadFactory( "ProducerPool_" + System.nanoTime() + ":" + model.getTableName() + ", range: " + model .getBucketId())); producerExecutorServiceTaskList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); LOGGER.debug("Initializing writer executors"); consumerExecutorService = Executors.newFixedThreadPool(1, new CarbonThreadFactory( "ConsumerPool_" + System.nanoTime() + ":" + model.getTableName() + ", range: " + model .getBucketId())); consumerExecutorServiceTaskList = new ArrayList<>(1); semaphore = new Semaphore(numberOfCores); tablePageList = new TablePageList(); // Start the consumer which will take each blocklet/page in order and write to a file Consumer consumer = new Consumer(tablePageList); consumerExecutorServiceTaskList.add(consumerExecutorService.submit(consumer)); }
public UnsafeIntermediateMerger(SortParameters parameters) { this.parameters = parameters; // processed file list this.rowPages = new ArrayList<UnsafeCarbonRowPage>(CarbonCommonConstants.CONSTANT_SIZE_TEN); this.mergedPages = new ArrayList<>(); this.executorService = Executors.newFixedThreadPool(parameters.getNumberOfCores(), new CarbonThreadFactory("UnsafeIntermediatePool:" + parameters.getTableName())); this.procFiles = new ArrayList<>(CarbonCommonConstants.CONSTANT_SIZE_TEN); this.mergerTask = new ArrayList<>(); Integer spillPercentage = CarbonProperties.getInstance().getSortMemorySpillPercentage(); this.spillSizeInSortMemory = UnsafeSortMemoryManager.INSTANCE.getUsableMemory() * spillPercentage / 100; // get memory chunk size long inMemoryChunkSizeInMB = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB(); if (spillSizeInSortMemory < inMemoryChunkSizeInMB * 1024 * 1024) { LOGGER.warn("the configure spill size is " + spillSizeInSortMemory + " less than the page size " + inMemoryChunkSizeInMB * 1024 * 1024 + ",so no merge and spill in-memory pages to disk"); } }
executorService = Executors.newCachedThreadPool(new CarbonThreadFactory( "DictionaryClientPool:" + configuration.getTableIdentifier().getCarbonTableIdentifier() .getTableName()));
.newFixedThreadPool(1, new CarbonThreadFactory("LocalFolderDeletionPool:" + tableName)); try { localFolderDeletionService.submit(new Callable<Void>() {
/** * This method will be used to initialize */ public void initialize() throws CarbonSortKeyAndGroupByException { // create holder list which will hold incoming rows // size of list will be sort buffer size + 1 to avoid creation of new // array in list array this.recordHolderList = new Object[sortBufferSize][]; // Delete if any older file exists in sort temp folder deleteSortLocationIfExists(); // create new sort temp directory CarbonDataProcessorUtil.createLocations(parameters.getTempFileLocation()); this.dataSorterAndWriterExecutorService = Executors .newFixedThreadPool(parameters.getNumberOfCores(), new CarbonThreadFactory("SortDataRowPool:" + parameters.getTableName())); semaphore = new Semaphore(parameters.getNumberOfCores()); }
/** * This method will be used to initialize */ public void initialize() throws MemoryException, CarbonSortKeyAndGroupByException { this.rowPage = createUnsafeRowPage(); // Delete if any older file exists in sort temp folder deleteSortLocationIfExists(); // create new sort temp directory CarbonDataProcessorUtil.createLocations(parameters.getTempFileLocation()); this.dataSorterAndWriterExecutorService = Executors .newFixedThreadPool(parameters.getNumberOfCores(), new CarbonThreadFactory("UnsafeSortDataRowPool:" + parameters.getTableName())); semaphore = new Semaphore(parameters.getNumberOfCores()); }
System.currentTimeMillis()); rangeExecutorService = Executors.newFixedThreadPool(iterators.length, new CarbonThreadFactory("WriterForwardPool: " + tableName)); List<Future<Void>> rangeExecutorServiceSubmitList = new ArrayList<>(iterators.length); int i = 0;
new CarbonThreadFactory("CompleteHDFSBackendPool:" + this.model.getTableName())); executorServiceSubmitList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); numberOfCores = model.getNumberOfCores() / 2; fallbackExecutorService = Executors.newFixedThreadPool(numberOfCores, new CarbonThreadFactory( "FallbackPool:" + model.getTableName() + ", range: " + model.getBucketId()));
@Override public void initialize() throws IOException { super.initialize(); rowParser = new RowParserImpl(getOutput(), configuration); executorService = Executors.newCachedThreadPool(new CarbonThreadFactory( "InputProcessorPool:" + configuration.getTableIdentifier().getCarbonTableIdentifier() .getTableName())); // if logger is enabled then raw data will be required. this.isRawDataRequired = CarbonDataProcessorUtil.isRawDataRequired(configuration); }
final DataLoadExecutor dataLoadExecutor = new DataLoadExecutor(); final ExecutorService executorService = Executors.newFixedThreadPool(1, new CarbonThreadFactory("CarbonRecordWriter:" + loadModel.getTableName()));
/** * Constructor to initialize * * @param tempFile * @param sortParameters * @param tableName */ public SortTempFileChunkHolder(File tempFile, SortParameters sortParameters, String tableName, boolean convertToActualField) { // set temp file this.tempFile = tempFile; this.readBufferSize = sortParameters.getBufferSize(); this.compressorName = sortParameters.getSortTempCompressorName(); this.tableFieldStat = new TableFieldStat(sortParameters); this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat); this.comparator = new IntermediateSortTempRowComparator( tableFieldStat.getIsSortColNoDictFlags(), tableFieldStat.getNoDictDataType()); this.executorService = Executors .newFixedThreadPool(1, new CarbonThreadFactory("SafeSortTempChunkHolderPool:" + tableName)); this.convertToActualField = convertToActualField; }
} else { executorService = Executors.newFixedThreadPool(iterators.length, new CarbonThreadFactory("NoSortDataWriterPool:" + configuration.getTableIdentifier() .getCarbonTableIdentifier().getTableName())); Future[] futures = new Future[iterators.length];
new CarbonThreadFactory("SafeParallelSorterPool:" + sortParameters.getTableName())); this.threadStatusObserver = new ThreadStatusObserver(executorService);
new CarbonThreadFactory("UnsafeParallelSorterPool:" + sortParameters.getTableName())); this.threadStatusObserver = new ThreadStatusObserver(executorService);