/** * This method will remove the cache for a given key * * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName, * tableName and columnIdentifier */ @Override public void invalidate( DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) { carbonLRUCache.remove( getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(), CacheType.REVERSE_DICTIONARY)); }
/** * This method will remove the cache for a given key * * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName, * tableName and columnIdentifier */ @Override public void invalidate( DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) { carbonLRUCache.remove( getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(), CacheType.FORWARD_DICTIONARY)); }
.put(dictionaryColumnUniqueIdentifiers.get(i).getColumnIdentifier().getColumnId(), columnDictionaryList.get(i));
/** * if file already exists then read metadata file and * validate the last entry end offset with file size. If * they are not equal that means some invalid data is present which needs * to be truncated * * @throws IOException if an I/O error occurs */ private void validateDictionaryFileOffsetWithLastSegmentEntryOffset() throws IOException { // read last dictionary chunk meta entry from dictionary metadata file chunkMetaObjectForLastSegmentEntry = getChunkMetaObjectForLastSegmentEntry(); int bytesToTruncate = 0; if (null != chunkMetaObjectForLastSegmentEntry) { bytesToTruncate = (int) (chunk_start_offset - chunkMetaObjectForLastSegmentEntry.getEnd_offset()); } if (bytesToTruncate > 0) { LOGGER.info("some inconsistency in dictionary file for column " + this.dictionaryColumnUniqueIdentifier.getColumnIdentifier()); // truncate the dictionary data till chunk meta end offset FileFactory.FileType fileType = FileFactory.getFileType(this.dictionaryFilePath); CarbonFile carbonFile = FileFactory.getCarbonFile(this.dictionaryFilePath, fileType); boolean truncateSuccess = carbonFile .truncate(this.dictionaryFilePath, chunkMetaObjectForLastSegmentEntry.getEnd_offset()); if (!truncateSuccess) { LOGGER.info("Diction file not truncated successfully for column " + this.dictionaryColumnUniqueIdentifier.getColumnIdentifier()); } } }
@Override public void clearAccessCount(List<DictionaryColumnUniqueIdentifier> keys) { for (DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier : keys) { Dictionary cacheable = (Dictionary) carbonLRUCache.get( getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(), CacheType.REVERSE_DICTIONARY)); cacheable.clear(); } }
@Override public void clearAccessCount(List<DictionaryColumnUniqueIdentifier> keys) { for (DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier : keys) { Dictionary cacheable = (Dictionary) carbonLRUCache.get( getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(), CacheType.FORWARD_DICTIONARY)); cacheable.clear(); } }
/** * This method will get the value for the given key. If value does not exist * for the given key, it will check and load the value. * * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName, * tableName and columnIdentifier * @return dictionary * @throws IOException in case memory is not sufficient to load dictionary into memory */ private Dictionary getDictionary( DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException { Dictionary forwardDictionary = null; // dictionary is only for primitive data type assert (!dictionaryColumnUniqueIdentifier.getDataType().isComplexType()); String columnIdentifier = dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(); ColumnDictionaryInfo columnDictionaryInfo = getColumnDictionaryInfo(dictionaryColumnUniqueIdentifier, columnIdentifier); // load sort index file in case of forward dictionary checkAndLoadDictionaryData(dictionaryColumnUniqueIdentifier, columnDictionaryInfo, getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(), CacheType.FORWARD_DICTIONARY), true); forwardDictionary = new ForwardDictionary(columnDictionaryInfo); return forwardDictionary; }
/** * This method will get the value for the given key. If value does not exist * for the given key, it will check and load the value. * * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName, * tableName and columnIdentifier * @return dictionary * @throws IOException in case memory is not sufficient to load dictionary into memory */ private Dictionary getDictionary( DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) throws IOException { Dictionary reverseDictionary = null; // dictionary is only for primitive data type assert (!dictionaryColumnUniqueIdentifier.getDataType().isComplexType()); String columnIdentifier = dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(); ColumnReverseDictionaryInfo columnReverseDictionaryInfo = getColumnReverseDictionaryInfo(dictionaryColumnUniqueIdentifier, columnIdentifier); // do not load sort index file for reverse dictionary checkAndLoadDictionaryData(dictionaryColumnUniqueIdentifier, columnReverseDictionaryInfo, getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(), CacheType.REVERSE_DICTIONARY), false); reverseDictionary = new ReverseDictionary(columnReverseDictionaryInfo); return reverseDictionary; }
CarbonFile dictFile = FileFactory.getCarbonFile(dictPath, FileFactory.getFileType(dictPath)); CarbonFile[] files = CarbonTablePath.getSortIndexFiles(dictFile.getParentFile(), dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId()); int maxTime; try {
/** * This method will return the value for the given key. It will not check and load * the data for the given key * * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName, * tableName and columnIdentifier * @return */ @Override public Dictionary getIfPresent( DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) { Dictionary forwardDictionary = null; ColumnDictionaryInfo columnDictionaryInfo = (ColumnDictionaryInfo) carbonLRUCache.get( getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(), CacheType.FORWARD_DICTIONARY)); if (null != columnDictionaryInfo) { forwardDictionary = new ForwardDictionary(columnDictionaryInfo); incrementDictionaryAccessCount(columnDictionaryInfo); } return forwardDictionary; }
/** * This method will return the value for the given key. It will not check and load * the data for the given key * * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName, * tableName and columnIdentifier * @return */ @Override public Dictionary getIfPresent( DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) { Dictionary reverseDictionary = null; ColumnReverseDictionaryInfo columnReverseDictionaryInfo = (ColumnReverseDictionaryInfo) carbonLRUCache.get( getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(), CacheType.REVERSE_DICTIONARY)); if (null != columnReverseDictionaryInfo) { reverseDictionary = new ReverseDictionary(columnReverseDictionaryInfo); incrementDictionaryAccessCount(columnReverseDictionaryInfo); } return reverseDictionary; }
+ this.dictionaryColumnUniqueIdentifier.getColumnIdentifier() + " at path " + this.dictionaryMetaFilePath); } finally {