private List<ExtendedBlocklet> pruneWithFilter(List<Segment> segments, FilterResolverIntf filterExp, List<PartitionSpec> partitions, List<ExtendedBlocklet> blocklets, Map<Segment, List<DataMap>> dataMaps) throws IOException { for (Segment segment : segments) { List<Blocklet> pruneBlocklets = new ArrayList<>(); SegmentProperties segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segment); for (DataMap dataMap : dataMaps.get(segment)) { pruneBlocklets.addAll(dataMap.prune(filterExp, segmentProperties, partitions)); } blocklets.addAll( addSegmentId(blockletDetailsFetcher.getExtendedBlocklets(pruneBlocklets, segment), segment.toString())); } return blocklets; }
@Override public Void call() throws IOException { Thread.currentThread().setName(threadName); for (SegmentDataMapGroup segmentDataMapGroup : segmentDataMapGroups) { List<Blocklet> pruneBlocklets = new ArrayList<>(); List<DataMap> dataMapList = dataMaps.get(segmentDataMapGroup.getSegment()); for (int i = segmentDataMapGroup.getFromIndex(); i <= segmentDataMapGroup.getToIndex(); i++) { pruneBlocklets.addAll(dataMapList.get(i).prune(filterExp, segmentPropertiesFetcher.getSegmentProperties(segmentDataMapGroup.getSegment()), partitions)); } synchronized (prunedBlockletMap) { List<Blocklet> pruneBlockletsExisting = prunedBlockletMap.get(segmentDataMapGroup.getSegment()); if (pruneBlockletsExisting != null) { pruneBlockletsExisting.addAll(pruneBlocklets); } else { prunedBlockletMap.put(segmentDataMapGroup.getSegment(), pruneBlocklets); } } } return null; } }));
/** * Pass the valid segments and prune the datamap using filter expression * * @param segments * @param filterExp * @return */ public List<ExtendedBlocklet> prune(List<Segment> segments, Expression filterExp, List<PartitionSpec> partitions) throws IOException { List<ExtendedBlocklet> blocklets = new ArrayList<>(); SegmentProperties segmentProperties; Map<Segment, List<DataMap>> dataMaps = dataMapFactory.getDataMaps(segments); for (Segment segment : segments) { List<Blocklet> pruneBlocklets = new ArrayList<>(); // if filter is not passed then return all the blocklets if (filterExp == null) { pruneBlocklets = blockletDetailsFetcher.getAllBlocklets(segment, partitions); } else { segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segment); for (DataMap dataMap : dataMaps.get(segment)) { pruneBlocklets .addAll(dataMap.prune(filterExp, segmentProperties, partitions, identifier)); } } blocklets.addAll(addSegmentId( blockletDetailsFetcher.getExtendedBlocklets(pruneBlocklets, segment), segment.toString())); } return blocklets; }
for (DataMap dataMap : dataMaps) { blocklets.addAll(dataMap.prune(filterExp, segmentPropertiesFetcher.getSegmentProperties(distributable.getSegment()), partitions));