/** * get executor and execute the query model. * * @param blockList * @return */ private CarbonIterator<RowBatch> executeBlockList(List<TableBlockInfo> blockList, String segmentId, String taskId, Configuration configuration) throws QueryExecutionException, IOException { queryModel.setTableBlockInfos(blockList); QueryStatisticsRecorder executorRecorder = CarbonTimeStatisticsFactory .createExecutorRecorder(queryModel.getQueryId() + "_" + segmentId + "_" + taskId); queryStatisticsRecorders.add(executorRecorder); queryModel.setStatisticsRecorder(executorRecorder); QueryExecutor queryExecutor = QueryExecutorFactory.getQueryExecutor(queryModel, configuration); queryExecutorList.add(queryExecutor); return queryExecutor.execute(queryModel); }
public void finish(CarbonFactHandler dataHandler) { CarbonTableIdentifier tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier(); String tableName = tableIdentifier.getTableName(); dataHandler.finish(); if (LOGGER.isDebugEnabled()) { LOGGER.debug("Record Processed For table: " + tableName); LOGGER.debug("Finished Carbon DataWriterProcessorStepImpl: Read: " + readCounter + ": Write: " + rowCounter.get()); } CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get()); processingComplete(dataHandler); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis()); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordMdkGenerateTotalTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis()); }
PartitionInfo partitionInfo, List<Integer> oldPartitionIdList) throws IOException { QueryStatisticsRecorder recorder = CarbonTimeStatisticsFactory.createDriverRecorder(); QueryStatistic statistic = new QueryStatistic();
/** * Below method will be used to process data to next step */ private boolean processRowToNextStep(UnsafeSortDataRows sortDataRows, SortParameters parameters) throws CarbonDataLoadingException { try { // start sorting sortDataRows.startSorting(); // check any more rows are present LOGGER.info("Record Processed For table: " + parameters.getTableName()); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis()); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis()); return false; } catch (Exception e) { throw new CarbonDataLoadingException(e); } }
if (null == queryProperties.queryStatisticsRecorder) { queryProperties.queryStatisticsRecorder = CarbonTimeStatisticsFactory.createExecutorRecorder(queryModel.getQueryId()); queryModel.setStatisticsRecorder(queryProperties.queryStatisticsRecorder);
private void finish(String tableName, CarbonFactHandler dataHandler) { CarbonDataWriterException exception = null; try { dataHandler.finish(); } catch (Exception e) { // if throw exception from here dataHandler will not be closed. // so just holding exception and later throwing exception LOGGER.error("Failed for table: " + tableName + " in finishing data handler", e); exception = new CarbonDataWriterException( "Failed for table: " + tableName + " in finishing data handler", e); } CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get()); try { processingComplete(dataHandler); } catch (Exception e) { if (null == exception) { exception = new CarbonDataWriterException(e); } } CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis()); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordMdkGenerateTotalTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis()); if (null != exception) { throw exception; } }
LOGGER.debug(logMessage); CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get()); try { processingComplete(dataHandler); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis()); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordMdkGenerateTotalTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis());
/** * Below method will be used to process data to next step */ private boolean processRowToNextStep(SortDataRows sortDataRows, SortParameters parameters) throws CarbonDataLoadingException { try { // start sorting sortDataRows.startSorting(); // check any more rows are present LOGGER.info("Record Processed For table: " + parameters.getTableName()); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis()); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis()); return false; } catch (CarbonSortKeyAndGroupByException e) { throw new CarbonDataLoadingException(e); } }
/** * Below method will be used to process data to next step */ private boolean processRowToNextStep(UnsafeSortDataRows[] sortDataRows, SortParameters parameters) throws CarbonDataLoadingException { try { for (int i = 0; i < sortDataRows.length; i++) { // start sorting sortDataRows[i].startSorting(); } // check any more rows are present LOGGER.info("Record Processed For table: " + parameters.getTableName()); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis()); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis()); return false; } catch (Exception e) { throw new CarbonDataLoadingException(e); } }
/** * Below method will be used to process data to next step */ private boolean processRowToNextStep(UnsafeSortDataRows sortDataRows, SortParameters parameters) throws CarbonDataLoadingException { try { // start sorting sortDataRows.startSorting(); // check any more rows are present LOGGER.info("Record Processed For table: " + parameters.getTableName()); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis()); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis()); return false; } catch (Exception e) { throw new CarbonDataLoadingException(e); } } }
/** * Below method will be used to process data to next step */ private boolean processRowToNextStep(SortDataRows[] sortDataRows, SortParameters parameters) throws CarbonDataLoadingException { try { for (int i = 0; i < sortDataRows.length; i++) { // start sorting sortDataRows[i].startSorting(); } // check any more rows are present LOGGER.info("Record Processed For table: " + parameters.getTableName()); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis()); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis()); return false; } catch (CarbonSortKeyAndGroupByException e) { throw new CarbonDataLoadingException(e); } }
@Override public void initialize() throws IOException { String nullFormat = configuration.getDataLoadProperty(DataLoadProcessorConstants.SERIALIZATION_NULL_FORMAT) .toString(); boolean isEmptyBadRecord = Boolean.parseBoolean( configuration.getDataLoadProperty(DataLoadProcessorConstants.IS_EMPTY_DATA_BAD_RECORD) .toString()); List<FieldConverter> fieldConverterList = new ArrayList<>(); localCaches = new Map[fields.length]; long lruCacheStartTime = System.currentTimeMillis(); DictionaryClient client = createDictionaryClient(); dictClients.add(client); for (int i = 0; i < fields.length; i++) { localCaches[i] = new ConcurrentHashMap<>(); FieldConverter fieldConverter = FieldEncoderFactory.getInstance() .createFieldEncoder(fields[i], configuration.getTableIdentifier(), i, nullFormat, client, configuration.getUseOnePass(), localCaches[i], isEmptyBadRecord, configuration.getParentTablePath(), isConvertToBinary); fieldConverterList.add(fieldConverter); } CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordLruCacheLoadTime((System.currentTimeMillis() - lruCacheStartTime) / 1000.0); fieldConverters = fieldConverterList.toArray(new FieldConverter[fieldConverterList.size()]); logHolder = new BadRecordLogHolder(); }
String tableName = tableIdentifier.getTableName(); try { CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis());
String tableName = tableIdentifier.getTableName(); try { CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis());
measureCount = configuration.getMeasureCount(); outputLength = measureCount + (this.noDictWithComplextCount > 0 ? 1 : 0) + 1; CarbonTimeStatisticsFactory.getLoadStatisticsInstance() .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis());