/** * create dataMap based on cache level * * @param carbonTable * @return */ public static DataMap createDataMap(CarbonTable carbonTable) { boolean cacheLevelBlock = BlockletDataMapUtil.isCacheLevelBlock(carbonTable); if (cacheLevelBlock) { // case1: when CACHE_LEVEL = BLOCK return new BlockDataMap(); } else { // case2: when CACHE_LEVEL = BLOCKLET return new BlockletDataMap(); } }
CarbonRowSchema[] schema = getFileFooterEntrySchema(); String filePath = getFilePath(); int numEntries = memoryDMStore.getRowCount(); int totalBlocklets = getTotalBlocklets(); int hitBlocklets = 0; if (filterExp == null) { for (int i = 0; i < numEntries; i++) { DataMapRow safeRow = memoryDMStore.getDataMapRow(schema, i).convertToSafeRow(); blocklets.add(createBlocklet(safeRow, getFileNameWithFilePath(safeRow, filePath), getBlockletId(safeRow), false)); .getFilterExecuterTree(filterExp, getSegmentProperties(), null, getMinMaxCacheColumns()); boolean useMinMaxForPruning = useMinMaxForExecutorPruning(filterExp); boolean[] minMaxFlag = getMinMaxFlag(safeRow, BLOCK_MIN_MAX_FLAG); String fileName = getFileNameWithFilePath(safeRow, filePath); short blockletId = getBlockletId(safeRow); boolean isValid = addBlockBasedOnMinMaxValue(filterExecuter, getMinMaxValue(safeRow, MAX_VALUES_INDEX), getMinMaxValue(safeRow, MIN_VALUES_INDEX), minMaxFlag, fileName, blockletId); if (isValid) { blocklets.add(createBlocklet(safeRow, fileName, blockletId, useMinMaxForPruning)); hitBlocklets += getBlockletNumOfEntry(entryIndex); ExplainCollector.setShowPruningInfo(true); ExplainCollector.addTotalBlocklets(totalBlocklets); ExplainCollector.addTotalBlocks(getTotalBlocks()); ExplainCollector.addDefaultDataMapPruningHit(hitBlocklets);
protected void setMinMaxFlagForTaskSummary(DataMapRow summaryRow, CarbonRowSchema[] taskSummarySchema, SegmentProperties segmentProperties, boolean[] minMaxFlag) { // add min max flag for all the dimension columns boolean[] minMaxFlagValuesForColumnsToBeCached = BlockletDataMapUtil .getMinMaxFlagValuesForColumnsToBeCached(segmentProperties, getMinMaxCacheColumns(), minMaxFlag); addMinMaxFlagValues(summaryRow, taskSummarySchema[TASK_MIN_MAX_FLAG], minMaxFlagValuesForColumnsToBeCached, TASK_MIN_MAX_FLAG); }
short totalBlockletsInOneBlock = 0; boolean isLastFileFooterEntryNeedToBeAdded = false; CarbonRowSchema[] schema = getFileFooterEntrySchema(); updateMinMaxFlag(fileFooter, minMaxFlag); updateMinMaxFlag(fileFooter, taskSummaryMinMaxFlag); previousDataFileFooter = fileFooter; totalBlockletsInOneBlock++; fileFooter.getBlockletIndex().getMinMaxIndex(); blockMinValues = compareAndUpdateMinMax(currentFooterMinMaxIndex.getMinValues(), blockMinValues, true); blockMaxValues = compareAndUpdateMinMax(currentFooterMinMaxIndex.getMaxValues(), blockMaxValues, false); updateMinMaxFlag(fileFooter, minMaxFlag); updateMinMaxFlag(fileFooter, taskSummaryMinMaxFlag); totalBlockletsInOneBlock++; TableBlockInfo previousBlockInfo = previousDataFileFooter.getBlockInfo().getTableBlockInfo(); summaryRow = loadToUnsafeBlock(schema, taskSummarySchema, previousDataFileFooter, segmentProperties, getMinMaxCacheColumns(), previousBlockInfo.getFilePath(), summaryRow, blockletDataMapInfo.getBlockMetaInfoMap().get(previousBlockInfo.getFilePath()), blockMinValues = fileFooter.getBlockletIndex().getMinMaxIndex().getMinValues(); blockMaxValues = fileFooter.getBlockletIndex().getMinMaxIndex().getMaxValues(); updateMinMaxFlag(fileFooter, minMaxFlag);
List<DataFileFooter> indexInfo) throws IOException, MemoryException { DataMapRowImpl summaryRow = null; CarbonRowSchema[] schema = getFileFooterEntrySchema(); boolean[] minMaxFlag = new boolean[segmentProperties.getColumnsValueSize().length]; Arrays.fill(minMaxFlag, true); byte[][] updatedMaxValues = CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, false); summaryRow = loadToUnsafeBlock(schema, taskSummarySchema, fileFooter, segmentProperties, getMinMaxCacheColumns(), blockInfo.getFilePath(), summaryRow, blockMetaInfo, updatedMinValues, updatedMaxValues, minMaxFlag); byte[] blockletCount = convertRowCountFromShortToByteArray(blockletCountList); summaryRow.setByteArray(blockletCount, taskSummarySchema.length - 1); setMinMaxFlagForTaskSummary(summaryRow, taskSummarySchema, segmentProperties, minMaxFlag); return summaryRow;
boolean[] minMaxFlagValuesForColumnsToBeCached = BlockletDataMapUtil .getMinMaxFlagValuesForColumnsToBeCached(segmentProperties, minMaxCacheColumns, minMaxFlag); row.setRow(addMinMax(schema[ordinal], minValuesForColumnsToBeCached), ordinal); addTaskMinMaxValues(summaryRow, taskSummarySchema, taskMinMaxOrdinal, minValuesForColumnsToBeCached, TASK_MIN_VALUES_INDEX, true); ordinal++; taskMinMaxOrdinal++; row.setRow(addMinMax(schema[ordinal], maxValuesForColumnsToBeCached), ordinal); addTaskMinMaxValues(summaryRow, taskSummarySchema, taskMinMaxOrdinal, maxValuesForColumnsToBeCached, TASK_MAX_VALUES_INDEX, false); ordinal++; getFileNameFromPath(filePath).getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS); row.setByteArray(filePathBytes, ordinal++); setLocations(blockMetaInfo.getLocationInfo(), row, ordinal++); addMinMaxFlagValues(row, schema[ordinal], minMaxFlagValuesForColumnsToBeCached, ordinal); memoryDMStore.addIndexRow(schema, row); } catch (Exception e) {
updatedMinMaxValues = minMaxValue; } else { byte[][] existingMinMaxValues = getMinMaxValue(taskMinMaxRow, ordinal); updatedMinMaxValues = compareAndUpdateMinMax(minMaxValue, existingMinMaxValues, isMinValueComparison);
private void finishWriting(CarbonRowSchema[] taskSummarySchema, byte[] filePath, byte[] fileName, byte[] segmentId, DataMapRowImpl summaryRow) throws MemoryException { if (memoryDMStore != null) { memoryDMStore.finishWriting(); } if (null != taskSummaryDMStore) { addTaskSummaryRowToUnsafeMemoryStore(taskSummarySchema, summaryRow, filePath, fileName, segmentId); taskSummaryDMStore.finishWriting(); } }
/** * The method clears the access count of table segments * * @param tableSegmentUniqueIdentifiersWrapper */ @Override public void clearAccessCount( List<TableBlockIndexUniqueIdentifierWrapper> tableSegmentUniqueIdentifiersWrapper) { for (TableBlockIndexUniqueIdentifierWrapper identifierWrapper : tableSegmentUniqueIdentifiersWrapper) { BlockDataMap cacheable = (BlockDataMap) lruCache.get( identifierWrapper.getTableBlockIndexUniqueIdentifier().getUniqueTableSegmentIdentifier()); cacheable.clear(); } } }
@Override public void put(TableBlockIndexUniqueIdentifierWrapper tableBlockIndexUniqueIdentifierWrapper, BlockletDataMapIndexWrapper wrapper) throws IOException, MemoryException { // As dataMap will use unsafe memory, it is not recommended to overwrite an existing entry // as in that case clearing unsafe memory need to be taken card. If at all datamap entry // in the cache need to be overwritten then use the invalidate interface // and then use the put interface if (null == getIfPresent(tableBlockIndexUniqueIdentifierWrapper)) { List<BlockDataMap> dataMaps = wrapper.getDataMaps(); try { for (BlockDataMap blockletDataMap : dataMaps) { blockletDataMap.convertToUnsafeDMStore(); } // Locking is not required here because in LRU cache map add method is synchronized to add // only one entry at a time and if a key already exists it will not overwrite the entry lruCache.put(tableBlockIndexUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier() .getUniqueTableSegmentIdentifier(), wrapper, wrapper.getMemorySize()); } catch (Throwable e) { // clear all the memory acquired by data map in case of any failure for (DataMap blockletDataMap : dataMaps) { blockletDataMap.clear(); } throw new IOException("Problem in adding datamap to cache.", e); } } }