public String getPath() { return getFilePath(); }
private CarbonInputSplit convertToCarbonInputSplit(ExtendedBlocklet blocklet) throws IOException { CarbonInputSplit split = CarbonInputSplit .from(blocklet.getSegmentId(), blocklet.getBlockletId(), new FileSplit(new Path(blocklet.getPath()), 0, blocklet.getLength(), blocklet.getLocations()), ColumnarFormatVersion.valueOf((short) blocklet.getDetailInfo().getVersionNumber()), blocklet.getDataMapWriterPath()); split.setDetailInfo(blocklet.getDetailInfo()); return split; }
protected ExtendedBlocklet createBlocklet(DataMapRow row, String fileName, short blockletId, boolean useMinMaxForPruning) { ExtendedBlocklet blocklet = new ExtendedBlocklet(fileName, blockletId + "", false); BlockletDetailInfo detailInfo = getBlockletDetailInfo(row, blockletId, blocklet); detailInfo.setBlockletInfoBinary(new byte[0]); blocklet.setDetailInfo(detailInfo); return blocklet; }
pathToRead.put(prunedBlocklet.getFilePath().replace('\\', '/'), prunedBlocklet); block.setDataMapWriterPath(pathToRead.get(block.getFilePath()).getDataMapWriterPath()); blockToRead.add(block);
/** * Prune the segments from the already pruned blocklets. * @param segments * @param prunedBlocklets */ private void pruneSegments(List<Segment> segments, List<ExtendedBlocklet> prunedBlocklets) { List<Segment> toBeRemovedSegments = new ArrayList<>(); for (Segment segment : segments) { boolean found = false; // Clear the old pruned index files if any present segment.getFilteredIndexShardNames().clear(); // Check the segment exist in any of the pruned blocklets. for (ExtendedBlocklet blocklet : prunedBlocklets) { if (blocklet.getSegmentId().equals(segment.toString())) { found = true; // Set the pruned index file to the segment for further pruning. String shardName = CarbonTablePath.getShardName(blocklet.getFilePath()); segment.setFilteredIndexShardName(shardName); } } // Add to remove segments list if not present in pruned blocklets. if (!found) { toBeRemovedSegments.add(segment); } } // Remove all segments which are already pruned from pruned blocklets segments.removeAll(toBeRemovedSegments); }
blockletMap.prune(filteredSegment, (FilterResolverIntf) null, partitions); for (ExtendedBlocklet blocklet : blocklets) { String blockName = blocklet.getPath(); blockName = CarbonTablePath.getCarbonDataFileName(blockName); blockName = blockName + CarbonTablePath.getCarbonDataExtension(); long rowCount = blocklet.getDetailInfo().getRowCount(); String segmentId = Segment.toSegment(blocklet.getSegmentId()).getSegmentNo(); String key = CarbonUpdateUtil.getSegmentBlockNameKey(segmentId, blockName);
/** * for explain command * get number of block by counting distinct file path of blocklets */ private int getBlockCount(List<ExtendedBlocklet> blocklets) { Set<String> filepaths = new HashSet<>(); for (ExtendedBlocklet blocklet: blocklets) { filepaths.add(blocklet.getPath()); } return filepaths.size(); }
@Override public List<ExtendedBlocklet> pruneBlocklets(List<ExtendedBlocklet> blocklets) throws IOException { List<ExtendedBlocklet> blockletList = new ArrayList<>(); for (ExtendedBlocklet blocklet: blocklets) { if (blocklet.getDataMapUniqueId().equals(uniqueId)) { blockletList.add(blocklet); } } return blockletList; }
.getTaskIdFromTaskNo(CarbonTablePath.DataFileUtil.getTaskNo(blocklet.getPath()));
@Override protected ExtendedBlocklet createBlocklet(DataMapRow row, String fileName, short blockletId, boolean useMinMaxForPruning) { if (isLegacyStore) { return super.createBlocklet(row, fileName, blockletId, useMinMaxForPruning); } ExtendedBlocklet blocklet = new ExtendedBlocklet(fileName, blockletId + ""); BlockletDetailInfo detailInfo = getBlockletDetailInfo(row, blockletId, blocklet); detailInfo.setColumnSchemas(getColumnSchema()); detailInfo.setBlockletInfoBinary(row.getByteArray(BLOCKLET_INFO_INDEX)); detailInfo.setPagesCount(row.getShort(BLOCKLET_PAGE_COUNT_INDEX)); detailInfo.setUseMinMaxForPruning(useMinMaxForPruning); blocklet.setDetailInfo(detailInfo); return blocklet; }