/** * Construct a CarbonReaderBuilder with table path and table name * * @param tablePath table path * @param tableName table name */ CarbonReaderBuilder(String tablePath, String tableName) { this.tablePath = tablePath; this.tableName = tableName; ThreadLocalSessionInfo.setCarbonSessionInfo(new CarbonSessionInfo()); }
private Object getOperationContext() { // when validate segments is disabled in thread local update it to CarbonTableInputFormat CarbonSessionInfo carbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo(); if (carbonSessionInfo != null) { return carbonSessionInfo.getThreadParams().getExtraInfo("partition.operationcontext"); } return null; }
/** * This method will initialize the thread pool size to be used for creating the * max number of threads for a job */ private void initThreadPoolSize() { thread_pool_size = CarbonProperties.getInstance().getNumberOfLoadingCores(); }
public static CarbonTaskInfo getCarbonTaskInfo() { if (null == threadLocal.get()) { CarbonTaskInfo carbonTaskInfo = new CarbonTaskInfo(); carbonTaskInfo.setTaskId(CarbonUtil.generateUUID()); ThreadLocalTaskInfo.setCarbonTaskInfo(carbonTaskInfo); } return threadLocal.get(); } }
private double getTotalTime(String partitionID) { this.totalTime = getLoadCsvfilesToDfTime() + getDicShuffleAndWriteFileTotalTime() + getLruCacheLoadTime() + getDictionaryValuesTotalTime(partitionID) + getDictionaryValue2MdkAdd2FileTime(partitionID); return totalTime; }
public static synchronized CarbonSessionInfo getOrCreateCarbonSessionInfo() { CarbonSessionInfo info = threadLocal.get(); if (info == null || info.getSessionParams() == null) { info = new CarbonSessionInfo(); info.setSessionParams(new SessionParams()); threadLocal.set(info); } return info; }
/** * Method to clear out the dictionary cache. */ @Override public void clear() { CarbonUtil.clearDictionaryCache(dictionary); }
@Override public void close() throws IOException { if (null != dataOutStream) { CarbonUtil.closeStream(dataOutStream); } }
public static Configuration getConfiguration() { Configuration conf; Object confObject = ThreadLocalSessionInfo.getOrCreateCarbonSessionInfo() .getNonSerializableExtraInfo().get("carbonConf"); if (confObject == null) { conf = configuration; } else { conf = (Configuration) confObject; } return conf; }
private static LoadStatistics genLoadStatisticsInstance() { if (loadStatisticsInstanceType.equalsIgnoreCase("true")) { return CarbonLoadStatisticsImpl.getInstance(); } else { return CarbonLoadStatisticsDummy.getInstance(); } }
/** * This method will validate and set the value for executor start up waiting time out */ private void validateDynamicSchedulerTimeOut() { validateRange(CARBON_DYNAMIC_ALLOCATION_SCHEDULER_TIMEOUT, CarbonCommonConstants.CARBON_DYNAMIC_ALLOCATION_SCHEDULER_TIMEOUT_DEFAULT, CarbonCommonConstants.CARBON_DYNAMIC_ALLOCATION_SCHEDULER_TIMEOUT_MIN, CarbonCommonConstants.CARBON_DYNAMIC_ALLOCATION_SCHEDULER_TIMEOUT_MAX); }
/** * Get short int value at rowId */ @Override public int getShortInt(int rowId) { return ByteUtil.valueOf3Bytes(shortIntData, rowId * 3); }
/** * initialize the value of dictionary chunk that can be kept in memory at a time */ private void initDictionaryChunkSize() { dictionary_one_chunk_size = CarbonUtil.getDictionaryChunkSize(); }
/** * Read column schema from binary * @param schemaArray * @throws IOException */ public void readColumnSchema(byte[] schemaArray) throws IOException { if (null != schemaArray) { columnSchemas = BlockletDataMapUtil.readColumnSchema(schemaArray); } }
/** * This method will search a given dimension and return the dimension from current block * * @param queryDimension * @return */ public CarbonDimension getDimensionFromCurrentBlock(CarbonDimension queryDimension) { return CarbonUtil.getDimensionFromCurrentBlock(this.dimensions, queryDimension); }
/** * The following methods convert byte array back to the real value. */ public static short toXorShort(byte[] bytes, int offset, final int length) { return (short) (toShort(bytes, offset, length) ^ Short.MIN_VALUE); }
/** * Number of cores to be used while compacting. * @return the number of cores to be used while compacting */ public int getNumberOfCompactingCores() { return getNumberOfCores(CarbonCommonConstants.NUM_CORES_COMPACTING); }
/** * Below method will be used to get the index info from index file * * @param filePath file path of the index file * @return list of index info * @throws IOException problem while reading the index file */ public List<DataFileFooter> getIndexInfo(String filePath, byte[] fileData) throws IOException { return getIndexInfo(filePath, fileData, true); }
/** * This method will initialize the thread pool size to be used for creating the * max number of threads for a job */ private void initThreadPoolSize() { thread_pool_size = CarbonProperties.getInstance().getNumberOfLoadingCores(); }
/** * Sort intermediate file size validation and if not valid then reset to the default value */ private void validateSortIntermediateFilesLimit() { validateRange(SORT_INTERMEDIATE_FILES_LIMIT, CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE, CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT_MIN, CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT_MAX); }