@Override public Iterator<CarbonRowBatch>[] execute() { int batchSize = CarbonProperties.getInstance().getBatchSize(); List<CarbonIterator<Object[]>>[] readerIterators = CarbonDataProcessorUtil.partitionInputReaderIterators(inputIterators, sdkWriterCores); Iterator<CarbonRowBatch>[] outIterators = new Iterator[readerIterators.length]; for (int i = 0; i < outIterators.length; i++) { outIterators[i] = new InputProcessorStepImpl.InputProcessorIterator(readerIterators[i], rowParser, batchSize, false, null, rowCounter, isRawDataRequired); } return outIterators; }
public SortDataRows(SortParameters parameters, SortIntermediateFileMerger intermediateFileMerger) { this.parameters = parameters; this.sortStepRowHandler = new SortStepRowHandler(parameters); this.intermediateFileMerger = intermediateFileMerger; int batchSize = CarbonProperties.getInstance().getBatchSize(); this.sortBufferSize = Math.max(parameters.getSortBufferSize(), batchSize); // observer of writing file in thread this.threadStatusObserver = new ThreadStatusObserver(); this.reUsableByteArrayDataOutputStream = new ThreadLocal<ReUsableByteArrayDataOutputStream>() { @Override protected ReUsableByteArrayDataOutputStream initialValue() { ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); return new ReUsableByteArrayDataOutputStream(byteStream); } }; }
@Override public Iterator<CarbonRowBatch>[] execute() { int batchSize = CarbonProperties.getInstance().getBatchSize(); List<CarbonIterator<Object[]>>[] readerIterators = CarbonDataProcessorUtil.partitionInputReaderIterators(inputIterators, sdkWriterCores); Iterator<CarbonRowBatch>[] outIterators = new Iterator[readerIterators.length]; for (int i = 0; i < outIterators.length; i++) { outIterators[i] = new InputProcessorIterator(readerIterators[i], rowParser, batchSize, configuration.isPreFetch(), executorService, rowCounter, isRawDataRequired); } return outIterators; }
@Override public Iterator<CarbonRowBatch>[] execute() { int batchSize = CarbonProperties.getInstance().getBatchSize(); List<CarbonIterator<Object[]>>[] readerIterators = CarbonDataProcessorUtil.partitionInputReaderIterators(this.inputIterators, sdkWriterCores); Iterator<CarbonRowBatch>[] outIterators = new Iterator[readerIterators.length]; for (int i = 0; i < outIterators.length; i++) { outIterators[i] = new InputProcessorIterator(readerIterators[i], batchSize, configuration.isPreFetch(), rowCounter, orderOfData, noDictionaryMapping, dataTypes, configuration, dataFieldsWithComplexDataType); } return outIterators; }
@Override public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException { this.executorService = Executors.newFixedThreadPool(iterators.length); this.threadStatusObserver = new ThreadStatusObserver(this.executorService); int batchSize = CarbonProperties.getInstance().getBatchSize(); final SortBatchHolder sortBatchHolder = new SortBatchHolder(sortParameters, iterators.length, this.threadStatusObserver); try { for (int i = 0; i < iterators.length; i++) { executorService.execute( new SortIteratorThread(iterators[i], sortBatchHolder, batchSize, rowCounter, this.threadStatusObserver)); } } catch (Exception e) { checkError(); throw new CarbonDataLoadingException("Problem while shutdown the server ", e); } checkError(); // Creates the iterator to read from merge sorter. Iterator<CarbonSortBatch> batchIterator = new CarbonIterator<CarbonSortBatch>() { @Override public boolean hasNext() { return sortBatchHolder.hasNext(); } @Override public CarbonSortBatch next() { return new CarbonSortBatch(sortBatchHolder.next()); } }; return new Iterator[] { batchIterator }; }
UnsafeSortDataRows sortDataRow = new UnsafeSortDataRows(sortParameters, unsafeIntermediateFileMerger, inMemoryChunkSizeInMB); final int batchSize = CarbonProperties.getInstance().getBatchSize(); try { sortDataRow.initialize();
throws CarbonDataLoadingException { SortDataRows sortDataRow = new SortDataRows(sortParameters, intermediateFileMerger); final int batchSize = CarbonProperties.getInstance().getBatchSize(); try { sortDataRow.initialize();
final int batchSize = CarbonProperties.getInstance().getBatchSize(); try { for (int i = 0; i < iterators.length; i++) {
final int batchSize = CarbonProperties.getInstance().getBatchSize(); try {