public static Long addDataIndexSizeIntoMetaEntry(LoadMetadataDetails loadMetadataDetails, String segmentId, CarbonTable carbonTable) throws IOException { Map<String, Long> dataIndexSize = CarbonUtil.getDataSizeAndIndexSize( carbonTable.getTablePath(), new Segment(segmentId, loadMetadataDetails.getSegmentFile())); Long dataSize = dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE); loadMetadataDetails.setDataSize(String.valueOf(dataSize)); Long indexSize = dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE); loadMetadataDetails.setIndexSize(String.valueOf(indexSize)); return dataSize + indexSize; }
/** * Get the segment object corresponding to segmentNo * @param segmentNo * @param loadMetadataDetails * @return */ public static Segment getSegment(String segmentNo, LoadMetadataDetails[] loadMetadataDetails) { for (LoadMetadataDetails details: loadMetadataDetails) { if (details.getLoadName().equals(segmentNo)) { return new Segment(details.getLoadName(), details.getSegmentFile()); } } return null; }
/** * Get the partition specs of the segment * @param segmentId * @param tablePath * @return * @throws IOException */ public static List<PartitionSpec> getPartitionSpecs(String segmentId, String tablePath) throws IOException { LoadMetadataDetails segEntry = null; LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(CarbonTablePath.getMetadataPath(tablePath)); for (LoadMetadataDetails entry : details) { if (entry.getLoadName().equals(segmentId)) { segEntry = entry; break; } } if (segEntry != null && segEntry.getSegmentFile() != null) { SegmentFileStore fileStore = new SegmentFileStore(tablePath, segEntry.getSegmentFile()); List<PartitionSpec> partitionSpecs = fileStore.getPartitionSpecs(); for (PartitionSpec spec : partitionSpecs) { spec.setUuid(segmentId + "_" + segEntry.getLoadStartTime()); } return partitionSpecs; } return null; }
&& segment.getSegmentFile() != null) { List<String> toBeDeletedIndexFiles = new ArrayList<>(); List<String> toBeDeletedDataFiles = new ArrayList<>(); new SegmentFileStore(table.getTablePath(), segment.getSegmentFile()); List<String> indexOrMergeFiles = fileStore .readIndexFiles(SegmentStatus.MARKED_FOR_DELETE, false, FileFactory.getConfiguration());
if (segment.getSegmentFile() != null) { } else { sizeOfOneSegmentAcrossPartition = CarbonUtil.getSizeOfSegment(carbonTable.getTablePath(), new Segment(segId, segment.getSegmentFile()));
/** * method to identify the segments qualified for merging in case of IUD Compaction. * * @param segments * @param carbonLoadModel * @return */ private static List<LoadMetadataDetails> identifySegmentsToBeMergedBasedOnIUD( List<LoadMetadataDetails> segments, CarbonLoadModel carbonLoadModel) { List<LoadMetadataDetails> validSegments = new ArrayList<>(segments.size()); AbsoluteTableIdentifier absoluteTableIdentifier = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier(); int numberUpdateDeltaFilesThreshold = CarbonProperties.getInstance().getNoUpdateDeltaFilesThresholdForIUDCompaction(); for (LoadMetadataDetails seg : segments) { if ((isSegmentValid(seg)) && checkUpdateDeltaFilesInSeg( new Segment(seg.getLoadName(), seg.getSegmentFile()), absoluteTableIdentifier, carbonLoadModel.getSegmentUpdateStatusManager(), numberUpdateDeltaFilesThreshold)) { validSegments.add(seg); } } return validSegments; }
Segment seg = new Segment(segment.getMergedLoadName(), segment.getSegmentFile(), readCommittedScope, segment); if (!listOfValidSegments.contains(seg)) { new Segment(segment.getLoadName(), segment.getSegmentFile(), readCommittedScope)); new Segment(segment.getLoadName(), segment.getSegmentFile(), readCommittedScope)); continue; new Segment(segment.getLoadName(), segment.getSegmentFile(), readCommittedScope, segment)); } else if ((SegmentStatus.LOAD_FAILURE == segment.getSegmentStatus() || SegmentStatus.COMPACTED == segment.getSegmentStatus() || SegmentStatus.MARKED_FOR_DELETE == segment.getSegmentStatus())) { listOfInvalidSegments.add(new Segment(segment.getLoadName(), segment.getSegmentFile())); } else if (SegmentStatus.INSERT_IN_PROGRESS == segment.getSegmentStatus() || SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == segment.getSegmentStatus()) { listOfInProgressSegments.add( new Segment(segment.getLoadName(), segment.getSegmentFile(), readCommittedScope));
if (checkIfLoadCanBeDeletedPhysically(oneLoad, isForceDelete)) { try { if (oneLoad.getSegmentFile() != null) { SegmentFileStore.deleteSegment(carbonTable.getAbsoluteTableIdentifier().getTablePath(), new Segment(oneLoad.getLoadName(), oneLoad.getSegmentFile()), specs, updateStatusManager); } else {
if (operationContext != null && carbonTable.hasAggregationDataMap()) { operationContext .setProperty("current.segmentfile", newMetaEntry.getSegmentFile()); LoadEvents.LoadTablePreStatusUpdateEvent event = new LoadEvents.LoadTablePreStatusUpdateEvent(carbonTable.getCarbonTableIdentifier(),