.get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED); totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED, totalPagesScanned.getCount() + dataBlock.numberOfPages()); if (null == dataBlock.getColumnsMaxValue() || null == dataBlock.getColumnsMinValue()) { return true; CarbonCommonConstants.FILE_SEPARATOR + dataBlock.blockletIndex(); bitSet = ((ImplicitColumnFilterExecutor) filterExecuter) .isFilterValuesPresentInBlockOrBlocklet( dataBlock.getColumnsMaxValue(), dataBlock.getColumnsMinValue(), blockletId, dataBlock.minMaxFlagArray()); } else { bitSet = this.filterExecuter .isScanRequired(dataBlock.getColumnsMaxValue(), dataBlock.getColumnsMinValue(), dataBlock.minMaxFlagArray());
/** * To get the next block * @return next data block * */ @Override public DataRefNode next() { // get the current blocks DataRefNode datablockTemp = datablock; // store the next data block datablock = datablock.getNextDataRefNode(); // increment the counter blockCounter++; // if all the data block is processed then // set the has next flag to false // or if number of blocks assigned to this iterator is processed // then also set the hasnext flag to false if (null == datablock || blockCounter >= this.totalNumberOfBlocksToScan) { hasNext = false; } return datablockTemp; } }
@Override public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, boolean useBitsetPipeline) { BitSetGroup bitSetGroup = new BitSetGroup( rawBlockletColumnChunks.getDataBlock().numberOfPages()); for (int i = 0; i < rawBlockletColumnChunks.getDataBlock().numberOfPages(); i++) { bitSetGroup.setBitSet( setBitSetForCompleteDimensionData( rawBlockletColumnChunks.getDataBlock().getPageRowCount(i)), i); } return bitSetGroup; }
/** * Selects the blocks based on col max and min value. * * @param listOfDataBlocksToScan * @param dataRefNode */ private void addBlockBasedOnMinMaxValue(FilterExecuter filterExecuter, List<DataRefNode> listOfDataBlocksToScan, DataRefNode dataRefNode, boolean[] isMinMaxSet) { if (null == dataRefNode.getColumnsMinValue() || null == dataRefNode.getColumnsMaxValue()) { listOfDataBlocksToScan.add(dataRefNode); return; } BitSet bitSet = filterExecuter .isScanRequired(dataRefNode.getColumnsMaxValue(), dataRefNode.getColumnsMinValue(), isMinMaxSet); if (!bitSet.isEmpty()) { listOfDataBlocksToScan.add(dataRefNode); } }
@Override public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, boolean useBitsetPipeLine) throws IOException { int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows(); return FilterUtil .createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(), numberOfRows, !isDefaultValuePresentInFilterValues); }
int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows(); return FilterUtil .createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(), numberOfRows, true); if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) { rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readDimensionChunk( rawBlockletColumnChunks.getFileReader(), chunkIndex); if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readMeasureChunk( rawBlockletColumnChunks.getFileReader(), chunkIndex);
@Override public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException { for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) { DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i); if (!dimColumnEvaluatorInfo.getDimension().getDataType().isComplexType()) { if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[i]]) { rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[i]] = rawBlockletColumnChunks.getDataBlock().readDimensionChunk( rawBlockletColumnChunks.getFileReader(), dimensionChunkIndex[i]); } } else { GenericQueryType complexType = complexDimensionInfoMap.get(dimensionChunkIndex[i]); complexType.fillRequiredBlockData(rawBlockletColumnChunks); } } for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) { int chunkIndex = msrColEvalutorInfoList.get(0).getColumnIndex(); if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock() .readMeasureChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex); } } } }
BitSetGroup fgBitSetGroup = rawBlockletColumnChunks.getDataBlock().getIndexedData(); rawBlockletColumnChunks.setBitSetGroup(fgBitSetGroup); scannedResult.setBlockletId( blockExecutionInfo.getBlockIdString() + CarbonCommonConstants.FILE_SEPARATOR + rawBlockletColumnChunks.getDataBlock().blockletIndex()); blockExecutionInfo.getAllSelectedDimensionColumnIndexRange(); DimensionRawColumnChunk[] projectionListDimensionChunk = rawBlockletColumnChunks.getDataBlock() .readDimensionChunks(fileReader, allSelectedDimensionColumnIndexRange); for (int[] columnIndexRange : allSelectedDimensionColumnIndexRange) { System.arraycopy(projectionListDimensionChunk, columnIndexRange[0], if (null == dimensionRawColumnChunks[projectionListDimensionIndex]) { dimensionRawColumnChunks[projectionListDimensionIndex] = rawBlockletColumnChunks.getDataBlock().readDimensionChunk( fileReader, projectionListDimensionIndex); blockExecutionInfo.getAllSelectedMeasureIndexRange(); MeasureRawColumnChunk[] projectionListMeasureChunk = rawBlockletColumnChunks.getDataBlock() .readMeasureChunks(fileReader, allSelectedMeasureColumnIndexRange); for (int[] columnIndexRange : allSelectedMeasureColumnIndexRange) { System.arraycopy(projectionListMeasureChunk, columnIndexRange[0], measureRawColumnChunks, if (null == measureRawColumnChunks[projectionListMeasureIndex]) { measureRawColumnChunks[projectionListMeasureIndex] = rawBlockletColumnChunks.getDataBlock() .readMeasureChunk(fileReader, projectionListMeasureIndex);
int numberOfPages = rawBlockletColumnChunks.getDataBlock().numberOfPages(); BitSet bitSet = new BitSet(numberOfPages); bitSet.set(0, numberOfPages); rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock() .readDimensionChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex); rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock() .readMeasureChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
blockExecutionInfo.getAllSelectedDimensionColumnIndexRange(); DimensionRawColumnChunk[] projectionListDimensionChunk = rawBlockletColumnChunks.getDataBlock() .readDimensionChunks(fileReader, allSelectedDimensionColumnIndexRange); for (int[] columnIndexRange : allSelectedDimensionColumnIndexRange) { for (int i = columnIndexRange[0]; i < columnIndexRange[1] + 1; i++) { dimLazyWrapperChunks[projectionListDimensionIndex].rawColumnChunk = rawBlockletColumnChunks.getDataBlock() .readDimensionChunk(fileReader, projectionListDimensionIndex); blockExecutionInfo.getAllSelectedMeasureIndexRange(); MeasureRawColumnChunk[] projectionListMeasureChunk = rawBlockletColumnChunks.getDataBlock() .readMeasureChunks(fileReader, allSelectedMeasureColumnIndexRange); for (int[] columnIndexRange : allSelectedMeasureColumnIndexRange) { for (int i = columnIndexRange[0]; i < columnIndexRange[1] + 1; i++) { msrLazyWrapperChunks[projectionListMeasureIndex].rawColumnChunk = rawBlockletColumnChunks.getDataBlock() .readMeasureChunk(fileReader, projectionListMeasureIndex);
.get(QueryStatisticsConstants.VALID_PAGE_SCANNED); validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED, validPages.getCount() + rawBlockletColumnChunks.getDataBlock().numberOfPages()); totalPagesScanned.getCount() + rawBlockletColumnChunks.getDataBlock().numberOfPages()); String blockletId = blockExecutionInfo.getBlockIdString() + CarbonCommonConstants.FILE_SEPARATOR + rawBlockletColumnChunks.getDataBlock().blockletIndex(); scannedResult.setBlockletId(blockletId); DimensionRawColumnChunk[] dimensionRawColumnChunks = DimensionColumnPage[][] dimensionColumnDataChunks = new DimensionColumnPage[dimensionRawColumnChunks.length][rawBlockletColumnChunks .getDataBlock().numberOfPages()]; MeasureRawColumnChunk[] measureRawColumnChunks = rawBlockletColumnChunks.getMeasureRawColumnChunks(); ColumnPage[][] measureColumnPages = new ColumnPage[measureRawColumnChunks.length][rawBlockletColumnChunks.getDataBlock() .numberOfPages()]; scannedResult.setDimensionColumnPages(dimensionColumnDataChunks); scannedResult.setMeasureColumnPages(measureColumnPages); numberOfRows = new int[rawBlockletColumnChunks.getDataBlock().numberOfPages()]; for (int i = 0; i < numberOfRows.length; i++) { numberOfRows[i] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i);
int numberOfRows = blockChunkHolder.getDataBlock().numRows(); return FilterUtil.createBitSetGroupWithDefaultValue( blockChunkHolder.getDataBlock().numberOfPages(), numberOfRows, true); blockChunkHolder.getDataBlock().readDimensionChunk( blockChunkHolder.getFileReader(), chunkIndex);
@Override public BitSet prunePages(RawBlockletColumnChunks rawBlockletColumnChunks) throws FilterUnsupportedException, IOException { int numberOfPages = rawBlockletColumnChunks.getDataBlock().numberOfPages(); BitSet bitSet = new BitSet(numberOfPages); bitSet.set(0, numberOfPages); return bitSet; }
int numberOfPages = blockChunkHolder.getDataBlock().numberOfPages(); BitSet bitSet = new BitSet(); bitSet.set(0, numberOfPages); .readDimensionChunk(blockChunkHolder.getFileReader(), chunkIndex);
for (int i = pages.nextSetBit(0); i >= 0; i = pages.nextSetBit(i + 1)) { pageFilteredPages[index] = i; numberOfRows[index++] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i); scannedResult.setBlockletId( blockExecutionInfo.getBlockIdString() + CarbonCommonConstants.FILE_SEPARATOR + rawBlockletColumnChunks.getDataBlock().blockletIndex());
private RawBlockletColumnChunks getNextBlockletColumnChunks() { RawBlockletColumnChunks rawBlockletColumnChunks = null; do { DataRefNode dataBlock = blockletIterator.next(); if (dataBlock.getColumnsMaxValue() == null || blockletScanner.isScanRequired(dataBlock)) { rawBlockletColumnChunks = RawBlockletColumnChunks.newInstance( blockExecutionInfo.getTotalNumberDimensionToRead(), blockExecutionInfo.getTotalNumberOfMeasureToRead(), fileReader, dataBlock); } } while (rawBlockletColumnChunks == null && blockletIterator.hasNext()); return rawBlockletColumnChunks; }
protected void readBlockDataChunk(RawBlockletColumnChunks blockChunkHolder) throws IOException { if (null == blockChunkHolder.getDimensionRawColumnChunks()[blockIndex]) { blockChunkHolder.getDimensionRawColumnChunks()[blockIndex] = blockChunkHolder.getDataBlock() .readDimensionChunk(blockChunkHolder.getFileReader(), blockIndex); } } }
int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows(); return FilterUtil .createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(), numberOfRows, true); if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) { rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readDimensionChunk( rawBlockletColumnChunks.getFileReader(), chunkIndex); if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readMeasureChunk( rawBlockletColumnChunks.getFileReader(), chunkIndex);
@Override public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException { if (isDimensionPresentInCurrentBlock) { int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping() .get(dimColumnEvaluatorInfo.getColumnIndex()); if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) { rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readDimensionChunk( rawBlockletColumnChunks.getFileReader(), chunkIndex); } } else if (isMeasurePresentInCurrentBlock) { int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping() .get(msrColumnEvaluatorInfo.getColumnIndex()); if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readMeasureChunk( rawBlockletColumnChunks.getFileReader(), chunkIndex); } } } }
int numberOfPages = rawBlockletColumnChunks.getDataBlock().numberOfPages(); BitSet bitSet = new BitSet(numberOfPages); bitSet.set(0, numberOfPages); rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock() .readDimensionChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex); rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock() .readMeasureChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);