private List<ExtendedBlocklet> intersectFilteredBlocklets(CarbonTable carbonTable, List<ExtendedBlocklet> previousDataMapPrunedBlocklets, List<ExtendedBlocklet> otherDataMapPrunedBlocklets) { List<ExtendedBlocklet> prunedBlocklets = null; if (BlockletDataMapUtil.isCacheLevelBlock(carbonTable)) { prunedBlocklets = new ArrayList<>(); for (ExtendedBlocklet otherBlocklet : otherDataMapPrunedBlocklets) { if (previousDataMapPrunedBlocklets.contains(otherBlocklet)) { prunedBlocklets.add(otherBlocklet); } } } else { prunedBlocklets = (List) CollectionUtils .intersection(otherDataMapPrunedBlocklets, previousDataMapPrunedBlocklets); } return prunedBlocklets; }
private void convertColumnSchemaToBinary() throws IOException { if (null != columnSchemas) { columnSchemaBinary = BlockletDataMapUtil.convertSchemaToBinary(columnSchemas); } }
/** * Method to get the min/max values for columns to be cached * * @param segmentProperties * @param minMaxCacheColumns * @param minMaxValuesForAllColumns * @return */ public static byte[][] getMinMaxForColumnsToBeCached(SegmentProperties segmentProperties, List<CarbonColumn> minMaxCacheColumns, byte[][] minMaxValuesForAllColumns) { byte[][] minMaxValuesForColumnsToBeCached = minMaxValuesForAllColumns; if (null != minMaxCacheColumns) { minMaxValuesForColumnsToBeCached = new byte[minMaxCacheColumns.size()][]; int counter = 0; for (CarbonColumn column : minMaxCacheColumns) { minMaxValuesForColumnsToBeCached[counter++] = minMaxValuesForAllColumns[getColumnOrdinal(segmentProperties, column)]; } } return minMaxValuesForColumnsToBeCached; }
if (carbonDataFileBlockMetaInfoMapping == null) { carbonDataFileBlockMetaInfoMapping = BlockletDataMapUtil.createCarbonDataFileBlockMetaInfoMapping(segmentFilePath, identifierWrapper.getConfiguration()); segInfoCache.put(segmentFilePath, carbonDataFileBlockMetaInfoMapping); .getBlockMetaInfoMap(identifierWrapper, indexFileStore, filesRead, carbonDataFileBlockMetaInfoMapping); BlockDataMap blockletDataMap = BlockletDataMapUtil.getIndexFileIdentifiersFromMergeFile(identifier, indexFileStore); for (TableBlockIndexUniqueIdentifier blockIndexUniqueIdentifier : tableBlockIndexUniqueIdentifiers) { Map<String, BlockMetaInfo> blockMetaInfoMap = BlockletDataMapUtil.getBlockMetaInfoMap( new TableBlockIndexUniqueIdentifierWrapper(blockIndexUniqueIdentifier, identifierWrapper.getCarbonTable()), indexFileStore, filesRead,
.getMinMaxForColumnsToBeCached(segmentProperties, minMaxCacheColumns, minMaxIndex.getMinValues()); byte[][] maxValuesForColumnsToBeCached = BlockletDataMapUtil .getMinMaxForColumnsToBeCached(segmentProperties, minMaxCacheColumns, minMaxIndex.getMaxValues()); boolean[] minMaxFlagValuesForColumnsToBeCached = BlockletDataMapUtil .getMinMaxFlagValuesForColumnsToBeCached(segmentProperties, minMaxCacheColumns, fileFooter.getBlockletIndex().getMinMaxIndex().getIsMinMaxSet()); row.setRow(addMinMax(schema[ordinal], minValuesForColumnsToBeCached), ordinal);
for (DataFileFooter footer : indexInfo) { if ((!isTransactionalTable) && (tableColumnList.size() != 0) && !isSameColumnSchemaList(footer.getColumnInTable(), tableColumnList)) { LOG.error("Schema of " + identifier.getIndexFileName() + " doesn't match with the table's schema"); BlockMetaInfo blockMetaInfo = createBlockMetaInfo(fileNameToMetaInfoMapping, blockPath);
protected void setMinMaxFlagForTaskSummary(DataMapRow summaryRow, CarbonRowSchema[] taskSummarySchema, SegmentProperties segmentProperties, boolean[] minMaxFlag) { // add min max flag for all the dimension columns boolean[] minMaxFlagValuesForColumnsToBeCached = BlockletDataMapUtil .getMinMaxFlagValuesForColumnsToBeCached(segmentProperties, getMinMaxCacheColumns(), minMaxFlag); addMinMaxFlagValues(summaryRow, taskSummarySchema[TASK_MIN_MAX_FLAG], minMaxFlagValuesForColumnsToBeCached, TASK_MIN_MAX_FLAG); }
continue; if (!filterColumnExistsInMinMaxColumnList(minMaxCacheColumns, filterDimension)) { serializeMinMax = true; break; if (!filterColumnExistsInMinMaxColumnList(minMaxCacheColumns, filterMeasure)) { serializeMinMax = true; break;
@Override public List<DataMapDistributable> getAllUncachedDistributables( List<DataMapDistributable> distributables) throws IOException { List<DataMapDistributable> distributablesToBeLoaded = new ArrayList<>(distributables.size()); for (DataMapDistributable distributable : distributables) { Segment segment = distributable.getSegment(); Set<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers = getTableBlockIndexUniqueIdentifiers(segment); // filter out the tableBlockIndexUniqueIdentifiers based on distributable TableBlockIndexUniqueIdentifier validIdentifier = BlockletDataMapUtil .filterIdentifiersBasedOnDistributable(tableBlockIndexUniqueIdentifiers, (BlockletDataMapDistributable) distributable); if (null == cache.getIfPresent( new TableBlockIndexUniqueIdentifierWrapper(validIdentifier, this.getCarbonTable()))) { ((BlockletDataMapDistributable) distributable) .setTableBlockIndexUniqueIdentifier(validIdentifier); distributablesToBeLoaded.add(distributable); } } return distributablesToBeLoaded; } }
.getMinMaxForColumnsToBeCached(segmentProperties, minMaxCacheColumns, minValues); byte[][] maxValuesForColumnsToBeCached = BlockletDataMapUtil .getMinMaxForColumnsToBeCached(segmentProperties, minMaxCacheColumns, maxValues); boolean[] minMaxFlagValuesForColumnsToBeCached = BlockletDataMapUtil .getMinMaxFlagValuesForColumnsToBeCached(segmentProperties, minMaxCacheColumns, minMaxFlag); row.setRow(addMinMax(schema[ordinal], minValuesForColumnsToBeCached), ordinal);
/** * create dataMap based on cache level * * @param carbonTable * @return */ public static DataMap createDataMap(CarbonTable carbonTable) { boolean cacheLevelBlock = BlockletDataMapUtil.isCacheLevelBlock(carbonTable); if (cacheLevelBlock) { // case1: when CACHE_LEVEL = BLOCK return new BlockDataMap(); } else { // case2: when CACHE_LEVEL = BLOCKLET return new BlockletDataMap(); } }
/** * Method to get the flag values for columns to be cached * * @param segmentProperties * @param minMaxCacheColumns * @param minMaxFlag * @return */ public static boolean[] getMinMaxFlagValuesForColumnsToBeCached( SegmentProperties segmentProperties, List<CarbonColumn> minMaxCacheColumns, boolean[] minMaxFlag) { boolean[] minMaxFlagValuesForColumnsToBeCached = minMaxFlag; if (null != minMaxCacheColumns) { minMaxFlagValuesForColumnsToBeCached = new boolean[minMaxCacheColumns.size()]; int counter = 0; for (CarbonColumn column : minMaxCacheColumns) { minMaxFlagValuesForColumnsToBeCached[counter++] = minMaxFlag[getColumnOrdinal(segmentProperties, column)]; } } return minMaxFlagValuesForColumnsToBeCached; }
/** * Method to fill the column ordinals to access based on the columns to be cached * * @param segmentProperties * @param minMaxCacheColumns * @return */ private static int[] getColumnOrdinalsToAccess(SegmentProperties segmentProperties, List<CarbonColumn> minMaxCacheColumns) { int[] columnOrdinalsTOAccess = null; if (null != minMaxCacheColumns) { columnOrdinalsTOAccess = new int[minMaxCacheColumns.size()]; int counter = 0; for (CarbonColumn column : minMaxCacheColumns) { columnOrdinalsTOAccess[counter++] = BlockletDataMapUtil.getColumnOrdinal(segmentProperties, column); } } else { // when columns to cache is not specified then column access order will be same as the array // index of min max length columnOrdinalsTOAccess = new int[segmentProperties.getColumnsValueSize().length]; for (int i = 0; i < columnOrdinalsTOAccess.length; i++) { columnOrdinalsTOAccess[i] = i; } } return columnOrdinalsTOAccess; } }
/** * Method to get the min max length of each column. It will return the length of only column * which will be cached * * @param segmentProperties * @param minMaxCacheColumns * @return */ private static int[] getMinMaxLength(SegmentProperties segmentProperties, List<CarbonColumn> minMaxCacheColumns) { int[] minMaxLen = null; if (null != minMaxCacheColumns) { minMaxLen = new int[minMaxCacheColumns.size()]; int counter = 0; for (CarbonColumn column : minMaxCacheColumns) { minMaxLen[counter++] = segmentProperties.getColumnsValueSize()[BlockletDataMapUtil .getColumnOrdinal(segmentProperties, column)]; } } else { minMaxLen = segmentProperties.getColumnsValueSize(); } return minMaxLen; }