/** * This method will log query result count and querytime * @param recordCount * @param recorder */ public void logStatistics(int recordCount, QueryStatisticsRecorder recorder) { // result size if (null != recorder) { QueryStatistic queryStatistic = new QueryStatistic(); queryStatistic.addCountStatistic(QueryStatisticsConstants.RESULT_SIZE, recordCount); recorder.recordStatistics(queryStatistic); } } }
private static QueryStatisticsRecorder genDriverRecorder() { if (driverRecorderType.equalsIgnoreCase("true")) { return DriverQueryStatisticsRecorderImpl.getInstance(); } else { return DriverQueryStatisticsRecorderDummy.getInstance(); } }
private void logStatistics(long queryStartTime) { if (!queryStatisticsRecorders.isEmpty()) { QueryStatistic queryStatistic = new QueryStatistic(); queryStatistic.addFixedTimeStatistic(QueryStatisticsConstants.EXECUTOR_PART, System.currentTimeMillis() - queryStartTime); for (QueryStatisticsRecorder recorder : queryStatisticsRecorders) { recorder.recordStatistics(queryStatistic); // print executor query statistics for each task_id recorder.logStatistics(); } } }
private void initQueryStatiticsModel() { this.queryStatisticsModel = new QueryStatisticsModel(); this.queryStatisticsModel.setRecorder(recorder); QueryStatistic queryStatisticTotalBlocklet = new QueryStatistic(); queryStatisticsModel.getStatisticsTypeAndObjMap() .put(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, queryStatisticTotalBlocklet); queryStatisticsModel.getRecorder().recordStatistics(queryStatisticTotalBlocklet); QueryStatistic queryStatisticValidScanBlocklet = new QueryStatistic(); queryStatisticsModel.getStatisticsTypeAndObjMap() .put(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, queryStatisticValidScanBlocklet); queryStatisticsModel.getRecorder().recordStatistics(queryStatisticValidScanBlocklet); QueryStatistic totalNumberOfPages = new QueryStatistic(); queryStatisticsModel.getStatisticsTypeAndObjMap() .put(QueryStatisticsConstants.TOTAL_PAGE_SCANNED, totalNumberOfPages); queryStatisticsModel.getRecorder().recordStatistics(totalNumberOfPages); QueryStatistic validPages = new QueryStatistic(); queryStatisticsModel.getStatisticsTypeAndObjMap() .put(QueryStatisticsConstants.VALID_PAGE_SCANNED, validPages); queryStatisticsModel.getRecorder().recordStatistics(validPages); QueryStatistic scannedPages = new QueryStatistic(); queryStatisticsModel.getStatisticsTypeAndObjMap() .put(QueryStatisticsConstants.PAGE_SCANNED, scannedPages); queryStatisticsModel.getRecorder().recordStatistics(scannedPages); QueryStatistic scanTime = new QueryStatistic(); queryStatisticsModel.getStatisticsTypeAndObjMap()
@Override public void readBlocklet(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException { long startTime = System.currentTimeMillis(); this.filterExecuter.readColumnChunks(rawBlockletColumnChunks); // adding statistics for carbon read time QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap() .get(QueryStatisticsConstants.READ_BLOCKlET_TIME); readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME, readTime.getCount() + (System.currentTimeMillis() - startTime)); }
public TaskStatistics build(long startTime, List<QueryStatistic> queryStatistics) { this.values[2] = startTime; for (QueryStatistic statistic : queryStatistics) { if (statistic.getMessage() != null) { for (int columnIndex = 3; columnIndex <= numOfColumns - 1; columnIndex++) { if (columns[columnIndex].comment.equals(statistic.getMessage())) { if (columns[columnIndex].isDuration) { values[columnIndex] += statistic.getTimeTaken(); } else { values[columnIndex] += statistic.getCount(); } break; } } } } return this; }
public TaskStatistics statisticsForTask(long taskId, long startTime) { try { return new TaskStatistics(queryId, taskId).build(startTime, queryStatistics); } catch (Exception ex) { LOGGER.error(ex); return null; } }
public static QueryStatisticsRecorder createExecutorRecorder(String queryId) { String queryStatisticsRecorderType = CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, CarbonCommonConstants.ENABLE_QUERY_STATISTICS_DEFAULT); if (queryStatisticsRecorderType.equalsIgnoreCase("true")) { return new QueryStatisticsRecorderImpl(queryId); } else { return new QueryStatisticsRecorderDummy(); } }
/** * Below method will be used to log the statistic */ public void logStatistics() { for (QueryStatistic statistic : queryStatistics) { LOGGER.log(StatisticLevel.STATISTIC, statistic.getStatistics(queryId)); } }
/** * Below method will be used to show statistic log as table */ public void logStatisticsForTask(TaskStatistics result) { if (null != result) { LOGGER.log(StatisticLevel.STATISTIC, "Print query statistic for each task id:" + "\n" + result.toString()); } }
@Override public String toString() { StringBuilder builder = new StringBuilder(); printStatisticTable(Arrays.asList(this), builder, ""); return builder.toString(); }
public static void printStatisticTable(List<TaskStatistics> stats, StringBuilder builder, String indent) { int numOfRows = stats.size(); int numOfColumns = columns.length; // header as string[] String[] header = new String[numOfColumns]; for (int columnIndex = 0; columnIndex < numOfColumns; columnIndex++) { header[columnIndex] = columns[columnIndex].name; } // convert rows to string[][] String[][] rows = new String[numOfRows][]; for (int rowIndex = 0; rowIndex < numOfRows; rowIndex++) { rows[rowIndex] = stats.get(rowIndex).convertValueToString(); } CarbonUtil.logTable(builder, header, rows, indent); }
private void fillMeasureData(BlockletScannedResult scannedResult, List<Object[]> listBasedResult) { long startTime = System.currentTimeMillis(); // if list is not empty after filling the dimension data then only fill the measure data if (!listBasedResult.isEmpty()) { fillMeasureDataBatch(listBasedResult, 1, scannedResult); } QueryStatistic measureFillingTime = queryStatisticsModel.getStatisticsTypeAndObjMap() .get(QueryStatisticsConstants.MEASURE_FILLING_TIME); measureFillingTime.addCountStatistic(QueryStatisticsConstants.MEASURE_FILLING_TIME, measureFillingTime.getCount() + (System.currentTimeMillis() - startTime)); }
private void fillMeasureData(BlockletScannedResult scannedResult, List<Object[]> listBasedResult) { long startTime = System.currentTimeMillis(); // if list is not empty after filling the dimension data then only fill the measure data if (!listBasedResult.isEmpty()) { fillMeasureDataBatch(listBasedResult, 1, scannedResult); } QueryStatistic measureFillingTime = queryStatisticsModel.getStatisticsTypeAndObjMap() .get(QueryStatisticsConstants.MEASURE_FILLING_TIME); measureFillingTime.addCountStatistic(QueryStatisticsConstants.MEASURE_FILLING_TIME, measureFillingTime.getCount() + (System.currentTimeMillis() - startTime)); }
private void fillMeasureData(BlockletScannedResult scannedResult, List<Object[]> listBasedResult) { long startTime = System.currentTimeMillis(); // if list is not empty after filling the dimension data then only fill the measure data if (!listBasedResult.isEmpty()) { fillMeasureDataBatch(listBasedResult, 1, scannedResult); } QueryStatistic measureFillingTime = queryStatisticsModel.getStatisticsTypeAndObjMap() .get(QueryStatisticsConstants.MEASURE_FILLING_TIME); measureFillingTime.addCountStatistic(QueryStatisticsConstants.MEASURE_FILLING_TIME, measureFillingTime.getCount() + (System.currentTimeMillis() - startTime)); } }
/** * This method will add a record both key and value to list object * it will keep track of how many record is processed, to handle limit scenario */ @Override public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) { long startTime = System.currentTimeMillis(); List<Object[]> listBasedResult = new ArrayList<>(batchSize); ProjectionMeasure[] queryMeasures = executionInfo.getProjectionMeasures(); // scan the record and add to list scanAndFillData(scannedResult, batchSize, listBasedResult, queryMeasures); QueryStatistic resultPrepTime = queryStatisticsModel.getStatisticsTypeAndObjMap() .get(QueryStatisticsConstants.RESULT_PREP_TIME); resultPrepTime.addCountStatistic(QueryStatisticsConstants.RESULT_PREP_TIME, resultPrepTime.getCount() + (System.currentTimeMillis() - startTime)); return listBasedResult; }
/** * This method will add a record both key and value to list object * it will keep track of how many record is processed, to handle limit scenario */ @Override public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) { long startTime = System.currentTimeMillis(); List<Object[]> listBasedResult = new ArrayList<>(batchSize); ProjectionMeasure[] queryMeasures = executionInfo.getProjectionMeasures(); // scan the record and add to list scanAndFillData(scannedResult, batchSize, listBasedResult, queryMeasures); QueryStatistic resultPrepTime = queryStatisticsModel.getStatisticsTypeAndObjMap() .get(QueryStatisticsConstants.RESULT_PREP_TIME); resultPrepTime.addCountStatistic(QueryStatisticsConstants.RESULT_PREP_TIME, resultPrepTime.getCount() + (System.currentTimeMillis() - startTime)); return listBasedResult; }
/** * This case is used only in case of compaction, since it does not use filter flow. */ public void fillDataChunks() { freeDataChunkMemory(); if (pageCounter >= pageFilteredRowCount.length) { return; } long startTime = System.currentTimeMillis(); for (int i = 0; i < dimensionColumnPages.length; i++) { if (dimensionColumnPages[i][pageCounter] == null && dimRawColumnChunks[i] != null) { dimensionColumnPages[i][pageCounter] = dimRawColumnChunks[i] .convertToDimColDataChunkWithOutCache(pageCounter, null); } } for (int i = 0; i < measureColumnPages.length; i++) { if (measureColumnPages[i][pageCounter] == null && msrRawColumnChunks[i] != null) { measureColumnPages[i][pageCounter] = msrRawColumnChunks[i] .convertToColumnPageWithOutCache(pageCounter, null); } } QueryStatistic pageUncompressTime = queryStatisticsModel.getStatisticsTypeAndObjMap() .get(QueryStatisticsConstants.PAGE_UNCOMPRESS_TIME); pageUncompressTime.addCountStatistic(QueryStatisticsConstants.PAGE_UNCOMPRESS_TIME, pageUncompressTime.getCount() + (System.currentTimeMillis() - startTime)); }
/** * This method will add a record both key and value to list object * it will keep track of how many record is processed, to handle limit scenario */ @Override public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) { long startTime = System.currentTimeMillis(); List<Object[]> listBasedResult = new ArrayList<>(batchSize); ProjectionMeasure[] queryMeasures = executionInfo.getActualQueryMeasures(); // scan the record and add to list scanAndFillData(scannedResult, batchSize, listBasedResult, queryMeasures); // re-fill dictionary and no dictionary key arrays for the newly added columns if (dimensionInfo.isDictionaryColumnAdded()) { fillDictionaryKeyArrayBatchWithLatestSchema(listBasedResult); } if (dimensionInfo.isNoDictionaryColumnAdded()) { fillNoDictionaryKeyArrayBatchWithLatestSchema(listBasedResult); } QueryStatistic resultPrepTime = queryStatisticsModel.getStatisticsTypeAndObjMap() .get(QueryStatisticsConstants.RESULT_PREP_TIME); resultPrepTime.addCountStatistic(QueryStatisticsConstants.RESULT_PREP_TIME, resultPrepTime.getCount() + (System.currentTimeMillis() - startTime)); return listBasedResult; }
@Override public void readBlocklet(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException { long startTime = System.currentTimeMillis(); DimensionRawColumnChunk[] dimensionRawColumnChunks = rawBlockletColumnChunks.getDataBlock() .readDimensionChunks(rawBlockletColumnChunks.getFileReader(), blockExecutionInfo.getAllSelectedDimensionColumnIndexRange()); rawBlockletColumnChunks.setDimensionRawColumnChunks(dimensionRawColumnChunks); MeasureRawColumnChunk[] measureRawColumnChunks = rawBlockletColumnChunks.getDataBlock() .readMeasureChunks(rawBlockletColumnChunks.getFileReader(), blockExecutionInfo.getAllSelectedMeasureIndexRange()); rawBlockletColumnChunks.setMeasureRawColumnChunks(measureRawColumnChunks); // adding statistics for carbon read time QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap() .get(QueryStatisticsConstants.READ_BLOCKlET_TIME); readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME, readTime.getCount() + (System.currentTimeMillis() - startTime)); }