/** * This method will initialize the thread pool size to be used for creating the * max number of threads for a job */ private void initThreadPoolSize() { thread_pool_size = CarbonProperties.getInstance().getNumberOfLoadingCores(); }
/** * This method will initialize the thread pool size to be used for creating the * max number of threads for a job */ private void initThreadPoolSize() { thread_pool_size = CarbonProperties.getInstance().getNumberOfLoadingCores(); }
@Override public void writeDictionaryData() { int numOfCores = CarbonProperties.getInstance().getNumberOfLoadingCores(); long start = System.currentTimeMillis(); ExecutorService executorService = Executors.newFixedThreadPool(numOfCores); for (final DictionaryGenerator generator : columnMap.values()) { executorService.execute(new WriteDictionaryDataRunnable(generator)); } try { executorService.shutdown(); executorService.awaitTermination(1, TimeUnit.HOURS); } catch (InterruptedException e) { LOGGER.error("Error loading the dictionary: " + e.getMessage()); } LOGGER.info("Total time taken to write dictionary file is: " + (System.currentTimeMillis() - start)); }
/** * Partition input iterators equally as per the number of threads. * * @return */ public static List<CarbonIterator<Object[]>>[] partitionInputReaderIterators( CarbonIterator<Object[]>[] inputIterators, short sdkWriterCores) { // Get the number of cores configured in property. int numberOfCores; if (sdkWriterCores > 0) { numberOfCores = sdkWriterCores; } else { numberOfCores = CarbonProperties.getInstance().getNumberOfLoadingCores(); } // Get the minimum of number of cores and iterators size to get the number of parallel threads // to be launched. int parallelThreadNumber = Math.min(inputIterators.length, numberOfCores); if (parallelThreadNumber <= 0) { parallelThreadNumber = 1; } List<CarbonIterator<Object[]>>[] iterators = new List[parallelThreadNumber]; for (int i = 0; i < parallelThreadNumber; i++) { iterators[i] = new ArrayList<>(); } // Equally partition the iterators as per number of threads for (int i = 0; i < inputIterators.length; i++) { iterators[i % parallelThreadNumber].add(inputIterators[i]); } return iterators; }
private void initNumberOfCores() { // in compaction flow the measure with decimal type will come as spark decimal. // need to convert it to byte array. if (this.isCompactionFlow()) { this.numberOfCores = CarbonProperties.getInstance().getNumberOfCompactingCores(); } else { this.numberOfCores = CarbonProperties.getInstance().getNumberOfLoadingCores(); } if (this.sortScope != null && this.sortScope.equals(SortScopeOptions.SortScope.GLOBAL_SORT)) { this.numberOfCores = 1; } // Overriding it to the task specified cores. if (this.getWritingCoresCount() > 0) { this.numberOfCores = this.getWritingCoresCount(); } }
configuration.setWritingCoresCount(loadModel.getSdkWriterCores()); configuration.setNumberOfLoadingCores(CarbonProperties.getInstance().getNumberOfLoadingCores());