@Override public RecordReader<Void, T> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { Configuration configuration = taskAttemptContext.getConfiguration(); QueryModel queryModel = createQueryModel(inputSplit, taskAttemptContext); CarbonReadSupport<T> readSupport = getReadSupportClass(configuration); return new CarbonRecordReader<T>(queryModel, readSupport, taskAttemptContext.getConfiguration()); }
CarbonInputFormat.setTableInfo(job.getConfiguration(), table.getTableInfo()); CarbonInputFormat.setTablePath(job.getConfiguration(), table.getTablePath()); CarbonInputFormat.setTableName(job.getConfiguration(), table.getTableName()); CarbonInputFormat.setDatabaseName(job.getConfiguration(), table.getDatabaseName()); CarbonInputFormat.setCarbonReadSupport(job.getConfiguration(), CarbonRowReadSupport.class); CarbonInputFormat .setColumnProjection(job.getConfiguration(), new CarbonProjection(projectColumns)); if (filter != null) { CarbonInputFormat.setFilterPredicates(job.getConfiguration(), filter);
DataMapJob dataMapJob) throws InvalidConfigurationException, IOException { CarbonTableInputFormat<V> format = new CarbonTableInputFormat<>(); CarbonInputFormat.setTablePath( conf, identifier.appendWithLocalPrefix(identifier.getTablePath())); CarbonInputFormat.setQuerySegment(conf, identifier); CarbonInputFormat.setFilterPredicates(conf, filterExpression); CarbonInputFormat.setColumnProjection(conf, columnProjection); if (dataMapJob != null) { DataMapUtil.setDataMapJob(conf, dataMapJob); String validateInputSegmentsKey = CarbonCommonConstants.VALIDATE_CARBON_INPUT_SEGMENTS + tableUniqueKey; CarbonInputFormat.setValidateSegmentsToAccess( conf, Boolean.valueOf(carbonSessionInfo.getThreadParams().getProperty( .getProperty(queryOnPreAggStreamingKey, "false")); String inputSegmentsKey = CarbonCommonConstants.CARBON_INPUT_SEGMENTS + tableUniqueKey; CarbonInputFormat.setValidateSegmentsToAccess(conf, Boolean.valueOf(carbonSessionInfo.getThreadParams() .getProperty(validateInputSegmentsKey, "true"))); CarbonInputFormat.setQuerySegment( conf, carbonSessionInfo.getThreadParams().getProperty( CarbonProperties.getInstance().getProperty(inputSegmentsKey, "*"))); if (queryOnPreAggStreaming) { CarbonInputFormat.setAccessStreamingSegments(conf, true);
public static <V> CarbonTableInputFormat<V> createCarbonTableInputFormat( Job job, CarbonTable carbonTable, String[] projectionColumns, Expression filterExpression, List<PartitionSpec> partitionNames, DataMapJob dataMapJob) throws IOException, InvalidConfigurationException { Configuration conf = job.getConfiguration(); CarbonInputFormat.setTableInfo(conf, carbonTable.getTableInfo()); CarbonInputFormat.setDatabaseName(conf, carbonTable.getTableInfo().getDatabaseName()); CarbonInputFormat.setTableName(conf, carbonTable.getTableInfo().getFactTable().getTableName()); if (partitionNames != null) { CarbonInputFormat.setPartitionsToPrune(conf, partitionNames); } CarbonInputFormat .setTransactionalTable(conf, carbonTable.getTableInfo().isTransactionalTable()); CarbonProjection columnProjection = new CarbonProjection(projectionColumns); return createInputFormat(conf, carbonTable.getAbsoluteTableIdentifier(), filterExpression, columnProjection, dataMapJob); }
CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP_DEFAULT)); DataMapJob dataMapJob = DataMapUtil.getDataMapJob(job.getConfiguration()); List<PartitionSpec> partitionsToPrune = getPartitionsToPrune(job.getConfiguration()); ExplainCollector.setDefaultDataMapPruningBlockHit(getBlockCount(prunedBlocklets)); DataMapChooser chooser = new DataMapChooser(getOrCreateCarbonTable(job.getConfiguration())); pruneSegments(segmentIds, prunedBlocklets); List<ExtendedBlocklet> cgPrunedBlocklets; prunedBlocklets = intersectFilteredBlocklets(carbonTable, prunedBlocklets, cgPrunedBlocklets); ExplainCollector.recordCGDataMapPruning( DataMapWrapperSimpleInfo.fromDataMapWrapper(cgDataMapExprWrapper), prunedBlocklets.size(), getBlockCount(prunedBlocklets)); if (isFgDataMapPruningEnable(job.getConfiguration()) && dataMapJob != null) { DataMapExprWrapper fgDataMapExprWrapper = chooser.chooseFGDataMap(resolver); if (fgDataMapExprWrapper != null) { pruneSegments(segmentIds, prunedBlocklets); List<ExtendedBlocklet> fgPrunedBlocklets = DataMapUtil.executeDataMapJob(carbonTable, resolver, segmentIds, fgDataMapExprWrapper, dataMapJob, partitionsToPrune); prunedBlocklets = intersectFilteredBlocklets(carbonTable, prunedBlocklets, fgPrunedBlocklets); ExplainCollector.recordFGDataMapPruning( DataMapWrapperSimpleInfo.fromDataMapWrapper(fgDataMapExprWrapper), prunedBlocklets.size(), getBlockCount(prunedBlocklets));
public QueryModel createQueryModel(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException { Configuration configuration = taskAttemptContext.getConfiguration(); CarbonTable carbonTable = getOrCreateCarbonTable(configuration); String projectionString = getColumnProjection(configuration); String[] projectColumns; if (projectionString != null) { .filterExpression(getFilterPredicates(configuration)) .dataConverter(getDataTypeConverter(configuration)) .build();
new Path[] { new Path(carbonTable.getTablePath()) }, job.getConfiguration()); List<ExtendedBlocklet> prunedBlocklets = getPrunedBlocklets(job, carbonTable, expression, segmentIds); CarbonInputSplit inputSplit = convertToCarbonInputSplit(blocklet); if (inputSplit != null) { resultFilteredBlocks.add(inputSplit);