@Override protected DateFormat initialValue() { DateFormat dateFormat = new SimpleDateFormat(CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)); dateFormat.setLenient(false); return dateFormat; } };
/** * This method will initialize the thread pool size to be used for creating the * max number of threads for a job */ private void initThreadPoolSize() { thread_pool_size = CarbonProperties.getInstance().getNumberOfLoadingCores(); }
public boolean isEnableAutoHandoff() { String enableAutoHandoffStr = CarbonProperties.getInstance().getProperty( ENABLE_AUTO_HANDOFF, CarbonCommonConstants.ENABLE_AUTO_HANDOFF_DEFAULT); return enableAutoHandoffStr.equalsIgnoreCase("true"); }
public HeapMemoryAllocator() { poolingThresholdBytes = CarbonProperties.getInstance().getHeapMemoryPoolingThresholdBytes(); boolean isDriver = Boolean.parseBoolean(CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE, CarbonCommonConstants.IS_DRIVER_INSTANCE_DEFAULT)); // if set 'poolingThresholdBytes' to -1 or the object creation call is in driver, // it should not go through the pooling mechanism. if (poolingThresholdBytes == -1 || isDriver) { shouldPooling = false; } }
/** * This method will initialize the thread pool size to be used for creating the * max number of threads for a job */ private void initThreadPoolSize() { thread_pool_size = CarbonProperties.getInstance().getNumberOfLoadingCores(); }
/** * Sets the batch size of records to read * * @param batch batch size * @return updated CarbonReaderBuilder */ public CarbonReaderBuilder withBatch(int batch) { CarbonProperties.getInstance() .addProperty(CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE, String.valueOf(batch)); return this; }
/** * This method will set the zookeeper status whether zookeeper to be used for locking or not. */ private static void getLockTypeConfigured() { lockTypeConfigured = CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.LOCK_TYPE_DEFAULT) .toUpperCase(); LOGGER.info("Configured lock type is: " + lockTypeConfigured); }
public CarbonInputSplit() { segment = null; taskId = "0"; bucketId = "0"; blockletId = "0"; numberOfBlocklets = 0; invalidSegments = new ArrayList<>(); version = CarbonProperties.getInstance().getFormatVersion(); }
/** * Return the store path */ public static String getStorePath() { return getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION); }
public CarbonInputSplit(String segmentId, Path path, long start, long length, String[] locations, String[] inMemoryHosts, FileFormat fileFormat) { super(path, start, length, locations, inMemoryHosts); this.segment = Segment.toSegment(segmentId); this.fileFormat = fileFormat; taskId = "0"; bucketId = "0"; blockletId = "0"; numberOfBlocklets = 0; invalidSegments = new ArrayList<>(); version = CarbonProperties.getInstance().getFormatVersion(); }
public static void setup() { boolean isQueryStatisticsEnabled = Boolean.parseBoolean(CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, CarbonCommonConstants.ENABLE_QUERY_STATISTICS_DEFAULT)); if (isQueryStatisticsEnabled) { INSTANCE = new ExplainCollector(); } }
public CarbonInputSplit(String segmentId, Path path, long start, long length, String[] locations, FileFormat fileFormat) { super(path, start, length, locations); this.segment = Segment.toSegment(segmentId); this.fileFormat = fileFormat; taskId = "0"; bucketId = "0"; blockletId = "0"; numberOfBlocklets = 0; invalidSegments = new ArrayList<>(); version = CarbonProperties.getInstance().getFormatVersion(); }
/** * get the default compressor. * This method can only be called in data load procedure to compress column page. * In query procedure, we should read the compressor information from the metadata * in datafiles when we want to decompress the content. */ public Compressor getCompressor() { String compressorType = CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.COMPRESSOR, CarbonCommonConstants.DEFAULT_COMPRESSOR); return getCompressor(compressorType); }
private static ICarbonLock getDataMapStatusLock() { return CarbonLockFactory .getSystemLevelCarbonLockObj(CarbonProperties.getInstance().getSystemFolderLocation(), LockUsage.DATAMAP_STATUS_LOCK); } }
@Override public void initialize(SortParameters sortParameters) { this.originSortParameters = sortParameters; int totalInMemoryChunkSizeInMB = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB(); inMemoryChunkSizeInMB = totalInMemoryChunkSizeInMB / columnRangeInfo.getNumOfRanges(); if (inMemoryChunkSizeInMB < 5) { inMemoryChunkSizeInMB = 5; } this.insideRowCounterList = new ArrayList<>(columnRangeInfo.getNumOfRanges()); for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) { insideRowCounterList.add(new AtomicLong(0)); } }
/** * check and initialize xxHash32 if enabled */ public ColumnReverseDictionaryInfo() { boolean useXXHash = Boolean.valueOf(CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.ENABLE_XXHASH, CarbonCommonConstants.ENABLE_XXHASH_DEFAULT)); if (useXXHash) { xxHash32 = XXHashFactory.fastestInstance().hash32(); } }
public BlockletDataHolder(ExecutorService fallbackpool, CarbonFactDataHandlerModel model) { encodedBlocklet = new EncodedBlocklet(fallbackpool, Boolean.parseBoolean( CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.LOCAL_DICTIONARY_DECODER_BASED_FALLBACK, CarbonCommonConstants.LOCAL_DICTIONARY_DECODER_BASED_FALLBACK_DEFAULT)), model.getColumnLocalDictGenMap()); }
@Override public Iterator<CarbonRowBatch>[] execute() { int batchSize = CarbonProperties.getInstance().getBatchSize(); List<CarbonIterator<Object[]>>[] readerIterators = CarbonDataProcessorUtil.partitionInputReaderIterators(inputIterators, sdkWriterCores); Iterator<CarbonRowBatch>[] outIterators = new Iterator[readerIterators.length]; for (int i = 0; i < outIterators.length; i++) { outIterators[i] = new InputProcessorStepImpl.InputProcessorIterator(readerIterators[i], rowParser, batchSize, false, null, rowCounter, isRawDataRequired); } return outIterators; }
public static void initialize() { String zooKeeperUrl = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ZOOKEEPER_URL); if (null == zk) { zk = ZookeeperInit.getInstance(zooKeeperUrl).getZookeeper(); } }
public static CarbonDataLoadConfiguration createConfiguration(CarbonLoadModel loadModel, String[] storeLocation) { CarbonDataProcessorUtil.createLocations(storeLocation); String databaseName = loadModel.getDatabaseName(); String tableName = loadModel.getTableName(); String tempLocationKey = CarbonDataProcessorUtil .getTempStoreLocationKey(databaseName, tableName, loadModel.getSegmentId(), loadModel.getTaskNo(), false, false); CarbonProperties.getInstance().addProperty(tempLocationKey, StringUtils.join(storeLocation, File.pathSeparator)); return createConfiguration(loadModel); }