/** * this methods clears the datamap of table from memory */ public void clearDataMaps(String tableUniqName) { List<TableDataMap> tableIndices = allDataMaps.get(tableUniqName); if (tableIndices != null) { for (TableDataMap tableDataMap : tableIndices) { if (tableDataMap != null) { // clear the segmentMap in BlockletDetailsFetcher,else the Segment will remain in executor // and the query fails as we will check whether the blocklet contains in the index or not tableDataMap.getBlockletDetailsFetcher().clear(); tableDataMap.clear(); } } } allDataMaps.remove(tableUniqName); tablePathMap.remove(tableUniqName); }
private List<ExtendedBlocklet> pruneWithoutFilter(List<Segment> segments, List<PartitionSpec> partitions, List<ExtendedBlocklet> blocklets) throws IOException { for (Segment segment : segments) { List<Blocklet> allBlocklets = blockletDetailsFetcher.getAllBlocklets(segment, partitions); blocklets.addAll( addSegmentId(blockletDetailsFetcher.getExtendedBlocklets(allBlocklets, segment), segment.toString())); } return blocklets; }
private List<ExtendedBlocklet> pruneWithFilter(List<Segment> segments, FilterResolverIntf filterExp, List<PartitionSpec> partitions, List<ExtendedBlocklet> blocklets, Map<Segment, List<DataMap>> dataMaps) throws IOException { for (Segment segment : segments) { List<Blocklet> pruneBlocklets = new ArrayList<>(); SegmentProperties segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segment); for (DataMap dataMap : dataMaps.get(segment)) { pruneBlocklets.addAll(dataMap.prune(filterExp, segmentProperties, partitions)); } blocklets.addAll( addSegmentId(blockletDetailsFetcher.getExtendedBlocklets(pruneBlocklets, segment), segment.toString())); } return blocklets; }
.getExtendedBlocklet(blocklet, distributable.getSegment()); if (dataMapFactory.getDataMapLevel() == DataMapLevel.FG) { String blockletwritePath =
try { blocklets.addAll(addSegmentId( blockletDetailsFetcher.getExtendedBlocklets(entry.getValue(), entry.getKey()), entry.getKey().toString())); } catch (IOException e) {
/** * Pass the valid segments and prune the datamap using filter expression * * @param segments * @param filterExp * @return */ public List<ExtendedBlocklet> prune(List<Segment> segments, Expression filterExp, List<PartitionSpec> partitions) throws IOException { List<ExtendedBlocklet> blocklets = new ArrayList<>(); SegmentProperties segmentProperties; Map<Segment, List<DataMap>> dataMaps = dataMapFactory.getDataMaps(segments); for (Segment segment : segments) { List<Blocklet> pruneBlocklets = new ArrayList<>(); // if filter is not passed then return all the blocklets if (filterExp == null) { pruneBlocklets = blockletDetailsFetcher.getAllBlocklets(segment, partitions); } else { segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segment); for (DataMap dataMap : dataMaps.get(segment)) { pruneBlocklets .addAll(dataMap.prune(filterExp, segmentProperties, partitions, identifier)); } } blocklets.addAll(addSegmentId( blockletDetailsFetcher.getExtendedBlocklets(pruneBlocklets, segment), segment.toString())); } return blocklets; }