@Override public List<ExtendedBlocklet> prune(DataMapDistributable distributable, List<PartitionSpec> partitionsToPrune) throws IOException { List<ExtendedBlocklet> leftPrune = left.prune(distributable, partitionsToPrune); List<ExtendedBlocklet> rightPrune = right.prune(distributable, partitionsToPrune); Set<ExtendedBlocklet> andBlocklets = new HashSet<>(); andBlocklets.addAll(leftPrune); andBlocklets.addAll(rightPrune); return new ArrayList<>(andBlocklets); }
@Override public List<DataMapDistributableWrapper> toDistributable(List<Segment> segments) throws IOException { List<DataMapDistributableWrapper> wrappers = new ArrayList<>(); wrappers.addAll(left.toDistributable(segments)); wrappers.addAll(right.toDistributable(segments)); return wrappers; }
@Override public DataMapLevel getDataMapLevel() { return left.getDataMapLevel(); }
List<DataMapDistributableWrapper> distributables = datamap.toDistributable(segments); List<ExtendedBlocklet> prunnedBlocklets = new LinkedList<ExtendedBlocklet>(); for (int i = 0; i < distributables.size(); i++) { DataMapDistributable dataMapDistributable = distributables.get(i).getDistributable(); prunnedBlocklets.addAll(datamap.prune(dataMapDistributable, null));
public static DataMapWrapperSimpleInfo fromDataMapWrapper(DataMapExprWrapper dataMapExprWrapper) { if (dataMapExprWrapper instanceof DataMapExprWrapperImpl) { return new DataMapWrapperSimpleInfo( ((DataMapExprWrapperImpl) dataMapExprWrapper).getDataMapSchema()); } else if (dataMapExprWrapper instanceof AndDataMapExprWrapper) { return new DataMapWrapperSimpleInfo(WrapperType.AND, fromDataMapWrapper(dataMapExprWrapper.getLeftDataMapWrapper()), fromDataMapWrapper(dataMapExprWrapper.getRightDataMapWrapprt())); } else { return new DataMapWrapperSimpleInfo(WrapperType.OR, fromDataMapWrapper(dataMapExprWrapper.getLeftDataMapWrapper()), fromDataMapWrapper(dataMapExprWrapper.getRightDataMapWrapprt())); } }
@Override public List<ExtendedBlocklet> pruneBlocklets(List<ExtendedBlocklet> blocklets) throws IOException { List<ExtendedBlocklet> leftPrune = left.pruneBlocklets(blocklets); List<ExtendedBlocklet> rightPrune = right.pruneBlocklets(blocklets); List<ExtendedBlocklet> andBlocklets = new ArrayList<>(); for (ExtendedBlocklet blocklet : leftPrune) { if (rightPrune.contains(blocklet)) { andBlocklets.add(blocklet); } } return andBlocklets; }
@Override public FilterResolverIntf getFilterResolverIntf(String uniqueId) { FilterResolverIntf leftExp = left.getFilterResolverIntf(uniqueId); FilterResolverIntf rightExp = right.getFilterResolverIntf(uniqueId); if (leftExp != null) { return leftExp; } else if (rightExp != null) { return rightExp; } return null; }
@Override public List<ExtendedBlocklet> pruneBlocklets(List<ExtendedBlocklet> blocklets) throws IOException { List<ExtendedBlocklet> leftPrune = left.pruneBlocklets(blocklets); List<ExtendedBlocklet> rightPrune = right.pruneBlocklets(blocklets); Set<ExtendedBlocklet> andBlocklets = new HashSet<>(); andBlocklets.addAll(leftPrune); andBlocklets.addAll(rightPrune); return new ArrayList<>(andBlocklets); }
@Override public FilterResolverIntf getFilterResolverIntf(String uniqueId) { FilterResolverIntf leftExp = left.getFilterResolverIntf(uniqueId); FilterResolverIntf rightExp = right.getFilterResolverIntf(uniqueId); if (leftExp != null) { return leftExp; } else if (rightExp != null) { return rightExp; } return null; }
@Override public List<ExtendedBlocklet> prune(List<Segment> segments, List<PartitionSpec> partitionsToPrune) throws IOException { List<ExtendedBlocklet> leftPrune = left.prune(segments, partitionsToPrune); List<ExtendedBlocklet> rightPrune = right.prune(segments, partitionsToPrune); List<ExtendedBlocklet> andBlocklets = new ArrayList<>(); for (ExtendedBlocklet blocklet : leftPrune) { if (rightPrune.contains(blocklet)) { andBlocklets.add(blocklet); } } return andBlocklets; }
@Override public List<DataMapDistributableWrapper> toDistributable(List<Segment> segments) throws IOException { List<DataMapDistributableWrapper> wrappers = new ArrayList<>(); wrappers.addAll(left.toDistributable(segments)); wrappers.addAll(right.toDistributable(segments)); return wrappers; }
/** * this method gets the datamapJob and call execute of that job, this will be launched for * distributed CG or FG * @return list of Extended blocklets after pruning */ public static List<ExtendedBlocklet> executeDataMapJob(CarbonTable carbonTable, FilterResolverIntf resolver, List<Segment> validSegments, DataMapExprWrapper dataMapExprWrapper, DataMapJob dataMapJob, List<PartitionSpec> partitionsToPrune) throws IOException { String className = "org.apache.carbondata.core.datamap.DistributableDataMapFormat"; SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegmentsInfo = getValidAndInvalidSegments(carbonTable, validSegments.get(0).getConfiguration()); List<Segment> invalidSegments = validAndInvalidSegmentsInfo.getInvalidSegments(); DistributableDataMapFormat dataMapFormat = createDataMapJob(carbonTable, dataMapExprWrapper, validSegments, invalidSegments, partitionsToPrune, className, false); List<ExtendedBlocklet> prunedBlocklets = dataMapJob.execute(dataMapFormat, resolver); // Apply expression on the blocklets. prunedBlocklets = dataMapExprWrapper.pruneBlocklets(prunedBlocklets); return prunedBlocklets; }
@Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { distributable = (DataMapDistributableWrapper) inputSplit; // clear the segmentMap and from cache in executor when there are invalid segments if (invalidSegments.size() > 0) { DataMapStoreManager.getInstance().clearInvalidSegments(table, invalidSegments); } TableDataMap tableDataMap = DataMapStoreManager.getInstance() .getDataMap(table, distributable.getDistributable().getDataMapSchema()); if (isJobToClearDataMaps) { // if job is to clear datamaps just clear datamaps from cache and return DataMapStoreManager.getInstance() .clearDataMaps(table.getCarbonTableIdentifier().getTableUniqueName()); // clear the segment properties cache from executor SegmentPropertiesAndSchemaHolder.getInstance() .invalidate(table.getAbsoluteTableIdentifier()); blockletIterator = Collections.emptyIterator(); return; } dataMaps = tableDataMap.getTableDataMaps(distributable.getDistributable()); List<ExtendedBlocklet> blocklets = tableDataMap .prune(dataMaps, distributable.getDistributable(), dataMapExprWrapper.getFilterResolverIntf(distributable.getUniqueId()), partitions); for (ExtendedBlocklet blocklet : blocklets) { blocklet.setDataMapUniqueId(distributable.getUniqueId()); } blockletIterator = blocklets.iterator(); }
@Override public DataMapLevel getDataMapLevel() { return left.getDataMapLevel(); }
@Override public List<ExtendedBlocklet> prune(DataMapDistributable distributable, List<PartitionSpec> partitionsToPrune) throws IOException { List<ExtendedBlocklet> leftPrune = left.prune(distributable, partitionsToPrune); List<ExtendedBlocklet> rightPrune = right.prune(distributable, partitionsToPrune); List<ExtendedBlocklet> andBlocklets = new ArrayList<>(); for (ExtendedBlocklet blocklet : leftPrune) { if (rightPrune.contains(blocklet)) { andBlocklets.add(blocklet); } } return andBlocklets; }
@Override public List<InputSplit> getSplits(JobContext job) throws IOException { List<DataMapDistributableWrapper> distributables = dataMapExprWrapper.toDistributable(validSegments); List<InputSplit> inputSplits = new ArrayList<>(distributables.size()); inputSplits.addAll(distributables); return inputSplits; }
@Override public List<ExtendedBlocklet> prune(List<Segment> segments, List<PartitionSpec> partitionsToPrune) throws IOException { List<ExtendedBlocklet> leftPrune = left.prune(segments, partitionsToPrune); List<ExtendedBlocklet> rightPrune = right.prune(segments, partitionsToPrune); Set<ExtendedBlocklet> andBlocklets = new HashSet<>(); andBlocklets.addAll(leftPrune); andBlocklets.addAll(rightPrune); return new ArrayList<>(andBlocklets); }
resolver, segmentIds, cgDataMapExprWrapper, dataMapJob, partitionsToPrune); } else { cgPrunedBlocklets = cgDataMapExprWrapper.prune(segmentIds, partitionsToPrune);