/** * Return the store path */ public static String getStorePath() { return getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION); }
/** * Sets the batch size of records to read * * @param batch batch size * @return updated CarbonReaderBuilder */ public CarbonReaderBuilder withBatch(int batch) { CarbonProperties.getInstance() .addProperty(CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE, String.valueOf(batch)); return this; }
public CarbonInputSplit() { segment = null; taskId = "0"; bucketId = "0"; blockletId = "0"; numberOfBlocklets = 0; invalidSegments = new ArrayList<>(); version = CarbonProperties.getInstance().getFormatVersion(); }
/** * This method will initialize the thread pool size to be used for creating the * max number of threads for a job */ private void initThreadPoolSize() { thread_pool_size = CarbonProperties.getInstance().getNumberOfLoadingCores(); }
public HeapMemoryAllocator() { poolingThresholdBytes = CarbonProperties.getInstance().getHeapMemoryPoolingThresholdBytes(); boolean isDriver = Boolean.parseBoolean(CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE, CarbonCommonConstants.IS_DRIVER_INSTANCE_DEFAULT)); // if set 'poolingThresholdBytes' to -1 or the object creation call is in driver, // it should not go through the pooling mechanism. if (poolingThresholdBytes == -1 || isDriver) { shouldPooling = false; } }
/** * Validate the specified property is positive integer value */ private void validatePositiveInteger(String propertyName) { String value = getInstance().getProperty(propertyName); try { int intValue = Integer.parseInt(value); if (intValue <= 0) { getInstance().removeProperty(propertyName); LOGGER.warn(String.format("The value \"%s\" configured for key \"%s\" " + "is invalid. Ignoring it", value, propertyName)); throw new IllegalArgumentException(); } } catch (NumberFormatException e) { getInstance().removeProperty(propertyName); LOGGER.warn(String.format("The value \"%s\" configured for key \"%s\" " + "is invalid. Ignoring it", value, propertyName)); throw e; } }
/** * This method will be used to get the properties value if property is not * present then it will return the default value * * @param key property key * @param defaultValue properties default value * @return properties value */ public String getProperty(String key, String defaultValue) { String value = getProperty(key); if (null == value) { return defaultValue; } return value; }
public ColumnarFormatVersion getFormatVersion() { String versionStr = getInstance().getProperty(CARBON_DATA_FILE_VERSION); if (versionStr == null) { return getDefaultFormatVersion(); } else { try { return ColumnarFormatVersion.valueOf(versionStr); } catch (IllegalArgumentException e) { return getDefaultFormatVersion(); } } }
private static ICarbonLock getDataMapStatusLock() { return CarbonLockFactory .getSystemLevelCarbonLockObj(CarbonProperties.getInstance().getSystemFolderLocation(), LockUsage.DATAMAP_STATUS_LOCK); } }
String tableName = loadModel.getTableName(); String tempLocationKey = databaseName + '_' + tableName + "_1"; CarbonProperties.getInstance().addProperty( tempLocationKey, storeLocation + "/" + databaseName + "/" + tableName); CarbonProperties.getInstance().addProperty("store_output_location", outPutLoc); CarbonProperties.getInstance().addProperty("send.signal.load", "false"); CarbonProperties.getInstance().addProperty("carbon.is.columnar.storage", "true"); CarbonProperties.getInstance().addProperty("carbon.dimension.split.value.in.columnar", "1"); CarbonProperties.getInstance().addProperty("carbon.is.fullyfilled.bits", "true"); CarbonProperties.getInstance().addProperty("is.int.based.indexer", "true"); CarbonProperties.getInstance().addProperty("aggregate.columnar.keyblock", "true"); CarbonProperties.getInstance().addProperty("is.compressed.keyblock", "false"); CSVInputFormat.setQuoteCharacter(configuration, loadModel.getQuoteChar()); CSVInputFormat.setReadBufferSize(configuration, CarbonProperties.getInstance().getProperty( CarbonCommonConstants.CSV_READ_BUFFER_SIZE, CarbonCommonConstants.CSV_READ_BUFFER_SIZE_DEFAULT));
/** * Get the configured system folder location. * @return */ public String getSystemFolderLocation() { String systemLocation = CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION); if (systemLocation == null) { systemLocation = getStorePath(); } if (systemLocation != null) { systemLocation = CarbonUtil.checkAndAppendFileSystemURIScheme(systemLocation); systemLocation = FileFactory.getUpdatedFilePath(systemLocation); } return systemLocation + CarbonCommonConstants.FILE_SEPARATOR + "_system"; }
@Override public Iterator<CarbonRowBatch>[] execute() { int batchSize = CarbonProperties.getInstance().getBatchSize(); List<CarbonIterator<Object[]>>[] readerIterators = CarbonDataProcessorUtil.partitionInputReaderIterators(inputIterators, sdkWriterCores); Iterator<CarbonRowBatch>[] outIterators = new Iterator[readerIterators.length]; for (int i = 0; i < outIterators.length; i++) { outIterators[i] = new InputProcessorStepImpl.InputProcessorIterator(readerIterators[i], rowParser, batchSize, false, null, rowCounter, isRawDataRequired); } return outIterators; }
/** * gettting the unmerged segment numbers to be merged. * * @return corrected value of unmerged segments to be merged */ public int[] getCompactionSegmentLevelCount() { String commaSeparatedLevels; commaSeparatedLevels = getProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD, CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD); int[] compactionSize = getIntArray(commaSeparatedLevels); if (0 == compactionSize.length) { compactionSize = getIntArray(CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD); } return compactionSize; }
private void initNumberOfCores() { // in compaction flow the measure with decimal type will come as spark decimal. // need to convert it to byte array. if (this.isCompactionFlow()) { this.numberOfCores = CarbonProperties.getInstance().getNumberOfCompactingCores(); } else { this.numberOfCores = CarbonProperties.getInstance().getNumberOfLoadingCores(); } if (this.sortScope != null && this.sortScope.equals(SortScopeOptions.SortScope.GLOBAL_SORT)) { this.numberOfCores = 1; } // Overriding it to the task specified cores. if (this.getWritingCoresCount() > 0) { this.numberOfCores = this.getWritingCoresCount(); } }
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); int[] noOfSegmentLevelsCount = CarbonProperties.getInstance() .getCompactionSegmentLevelCount(); noOfSegmentLevelsCount = CarbonProperties.getInstance() .getIntArray(tblProps.get(CarbonCommonConstants.TABLE_COMPACTION_LEVEL_THRESHOLD)); if (0 == noOfSegmentLevelsCount.length) { noOfSegmentLevelsCount = CarbonProperties.getInstance().getCompactionSegmentLevelCount();
public UnsafeIntermediateMerger(SortParameters parameters) { this.parameters = parameters; // processed file list this.rowPages = new ArrayList<UnsafeCarbonRowPage>(CarbonCommonConstants.CONSTANT_SIZE_TEN); this.mergedPages = new ArrayList<>(); this.executorService = Executors.newFixedThreadPool(parameters.getNumberOfCores(), new CarbonThreadFactory("UnsafeIntermediatePool:" + parameters.getTableName())); this.procFiles = new ArrayList<>(CarbonCommonConstants.CONSTANT_SIZE_TEN); this.mergerTask = new ArrayList<>(); Integer spillPercentage = CarbonProperties.getInstance().getSortMemorySpillPercentage(); this.spillSizeInSortMemory = UnsafeSortMemoryManager.INSTANCE.getUsableMemory() * spillPercentage / 100; // get memory chunk size long inMemoryChunkSizeInMB = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB(); if (spillSizeInSortMemory < inMemoryChunkSizeInMB * 1024 * 1024) { LOGGER.warn("the configure spill size is " + spillSizeInSortMemory + " less than the page size " + inMemoryChunkSizeInMB * 1024 * 1024 + ",so no merge and spill in-memory pages to disk"); } }
@Override public void initialize(SortParameters sortParameters) { this.originSortParameters = sortParameters; int totalInMemoryChunkSizeInMB = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB(); inMemoryChunkSizeInMB = totalInMemoryChunkSizeInMB / columnRangeInfo.getNumOfRanges(); if (inMemoryChunkSizeInMB < 5) { inMemoryChunkSizeInMB = 5; } this.insideRowCounterList = new ArrayList<>(columnRangeInfo.getNumOfRanges()); for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) { insideRowCounterList.add(new AtomicLong(0)); } }
break; case COMPACTION_SEGMENT_LEVEL_THRESHOLD: int[] values = CarbonProperties.getInstance().getIntArray(value); if (values.length != 2) { throw new InvalidConfigurationException(
/** * This method will create the sort parameters VO object * * @return */ private SortParameters createSortParameters() { int numberOfCompactingCores = CarbonProperties.getInstance().getNumberOfCompactingCores(); return SortParameters .createSortParameters(carbonTable, carbonLoadModel.getDatabaseName(), tableName, dimensionColumnCount, segmentProperties.getComplexDimensions().size(), measureCount, noDictionaryCount, segmentId, carbonLoadModel.getTaskNo(), noDictionaryColMapping, sortColumnMapping, isVarcharDimMapping, true, numberOfCompactingCores / 2); }
final String localStoreLocations = CarbonProperties.getInstance().getProperty(tempLocationKey); if (localStoreLocations == null) { throw new RuntimeException("Store location not set for the key " + tempLocationKey); CarbonProperties.getInstance().removeProperty(tempLocationKey); if (null != localFolderDeletionService) { localFolderDeletionService.shutdown();