@Override public void initialize(SortParameters sortParameters) { this.originSortParameters = sortParameters; int totalInMemoryChunkSizeInMB = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB(); inMemoryChunkSizeInMB = totalInMemoryChunkSizeInMB / columnRangeInfo.getNumOfRanges(); if (inMemoryChunkSizeInMB < 5) { inMemoryChunkSizeInMB = 5; } this.insideRowCounterList = new ArrayList<>(columnRangeInfo.getNumOfRanges()); for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) { insideRowCounterList.add(new AtomicLong(0)); } }
public UnsafeIntermediateMerger(SortParameters parameters) { this.parameters = parameters; // processed file list this.rowPages = new ArrayList<UnsafeCarbonRowPage>(CarbonCommonConstants.CONSTANT_SIZE_TEN); this.mergedPages = new ArrayList<>(); this.executorService = Executors.newFixedThreadPool(parameters.getNumberOfCores(), new CarbonThreadFactory("UnsafeIntermediatePool:" + parameters.getTableName())); this.procFiles = new ArrayList<>(CarbonCommonConstants.CONSTANT_SIZE_TEN); this.mergerTask = new ArrayList<>(); Integer spillPercentage = CarbonProperties.getInstance().getSortMemorySpillPercentage(); this.spillSizeInSortMemory = UnsafeSortMemoryManager.INSTANCE.getUsableMemory() * spillPercentage / 100; // get memory chunk size long inMemoryChunkSizeInMB = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB(); if (spillSizeInSortMemory < inMemoryChunkSizeInMB * 1024 * 1024) { LOGGER.warn("the configure spill size is " + spillSizeInSortMemory + " less than the page size " + inMemoryChunkSizeInMB * 1024 * 1024 + ",so no merge and spill in-memory pages to disk"); } }
private void createSortDataRows() { // For each batch, createSortDataRows() will be called. // Files saved to disk during sorting of previous batch,should not be considered // for this batch. // Hence use batchID as rangeID field of sorttempfiles. // so getFilesToMergeSort() will select only this batch files. this.sortParameters.setRangeId(batchId.incrementAndGet()); int inMemoryChunkSizeInMB = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB(); setTempLocation(sortParameters); this.finalMerger = new UnsafeSingleThreadFinalSortFilesMerger(sortParameters, sortParameters.getTempFileLocation()); unsafeIntermediateFileMerger = new UnsafeIntermediateMerger(sortParameters); sortDataRow = new UnsafeSortDataRows(sortParameters, unsafeIntermediateFileMerger, inMemoryChunkSizeInMB); try { sortDataRow.initialize(); } catch (Exception e) { throw new CarbonDataLoadingException(e); } batchCount++; }
@Override public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException { int inMemoryChunkSizeInMB = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB(); UnsafeSortDataRows sortDataRow = new UnsafeSortDataRows(sortParameters, unsafeIntermediateFileMerger, inMemoryChunkSizeInMB);