private static boolean isSegmentValid(LoadMetadataDetails seg) { return seg.getSegmentStatus() == SegmentStatus.SUCCESS || seg.getSegmentStatus() == SegmentStatus.LOAD_PARTIAL_SUCCESS || seg.getSegmentStatus() == SegmentStatus.MARKED_FOR_UPDATE; }
private static boolean isLoadDeletionRequired(String metaDataLocation) { LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(metaDataLocation); if (details != null && details.length > 0) { for (LoadMetadataDetails oneRow : details) { if ((SegmentStatus.MARKED_FOR_DELETE == oneRow.getSegmentStatus() || SegmentStatus.COMPACTED == oneRow.getSegmentStatus() || SegmentStatus.INSERT_IN_PROGRESS == oneRow.getSegmentStatus() || SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == oneRow.getSegmentStatus()) && oneRow.getVisibility().equalsIgnoreCase("true")) { return true; } } } return false; }
private static boolean checkIfLoadCanBeDeleted(LoadMetadataDetails oneLoad, boolean isForceDelete) { if ((SegmentStatus.MARKED_FOR_DELETE == oneLoad.getSegmentStatus() || SegmentStatus.COMPACTED == oneLoad.getSegmentStatus() || SegmentStatus.INSERT_IN_PROGRESS == oneLoad.getSegmentStatus() || SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == oneLoad.getSegmentStatus()) && oneLoad.getVisibility().equalsIgnoreCase("true")) { if (isForceDelete) { return true; } long deletionTime = oneLoad.getModificationOrdeletionTimesStamp(); return CarbonUpdateUtil.isMaxQueryTimeoutExceeded(deletionTime); } return false; }
private static boolean checkIfLoadCanBeDeletedPhysically(LoadMetadataDetails oneLoad, boolean isForceDelete) { if ((SegmentStatus.MARKED_FOR_DELETE == oneLoad.getSegmentStatus() || SegmentStatus.COMPACTED == oneLoad.getSegmentStatus())) { if (isForceDelete) { return true; } long deletionTime = oneLoad.getModificationOrdeletionTimesStamp(); return CarbonUpdateUtil.isMaxQueryTimeoutExceeded(deletionTime); } return false; }
/** * updates table status details using latest metadata * * @param oldMetadata * @param newMetadata * @return */ public static List<LoadMetadataDetails> updateLatestTableStatusDetails( LoadMetadataDetails[] oldMetadata, LoadMetadataDetails[] newMetadata) { List<LoadMetadataDetails> newListMetadata = new ArrayList<LoadMetadataDetails>(Arrays.asList(newMetadata)); for (LoadMetadataDetails oldSegment : oldMetadata) { if (SegmentStatus.MARKED_FOR_DELETE == oldSegment.getSegmentStatus()) { updateSegmentMetadataDetails(newListMetadata.get(newListMetadata.indexOf(oldSegment))); } } return newListMetadata; }
for (LoadMetadataDetails segment : listOfSegmentsBelowThresholdSize) { if (segment.getSegmentStatus() == SegmentStatus.STREAMING || segment.getSegmentStatus() == SegmentStatus.STREAMING_FINISH) { continue; for (LoadMetadataDetails segment : listOfSegmentsBelowThresholdSize) { if (segment.getSegmentStatus() == SegmentStatus.STREAMING || segment.getSegmentStatus() == SegmentStatus.STREAMING_FINISH) { continue;
/** * updates segment status and modificaton time details * * @param loadMetadata */ public static void updateSegmentMetadataDetails(LoadMetadataDetails loadMetadata) { // update status only if the segment is not marked for delete if (SegmentStatus.MARKED_FOR_DELETE != loadMetadata.getSegmentStatus()) { loadMetadata.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE); loadMetadata.setModificationOrdeletionTimesStamp(CarbonUpdateUtil.readCurrentTime()); } }
segmentStatus = loadMetadata.getSegmentStatus(); if (SegmentStatus.COMPACTED == segmentStatus) {
if (SegmentStatus.SUCCESS == segment.getSegmentStatus() || SegmentStatus.MARKED_FOR_UPDATE == segment.getSegmentStatus() || SegmentStatus.LOAD_PARTIAL_SUCCESS == segment.getSegmentStatus() || SegmentStatus.STREAMING == segment.getSegmentStatus() || SegmentStatus.STREAMING_FINISH == segment.getSegmentStatus()) { if (SegmentStatus.MARKED_FOR_UPDATE == segment.getSegmentStatus()) { listOfValidUpdatedSegments.add(seg); if (SegmentStatus.MARKED_FOR_UPDATE == segment.getSegmentStatus()) { if (SegmentStatus.STREAMING == segment.getSegmentStatus() || SegmentStatus.STREAMING_FINISH == segment.getSegmentStatus()) { listOfStreamSegments.add( new Segment(segment.getLoadName(), segment.getSegmentFile(), readCommittedScope)); new Segment(segment.getLoadName(), segment.getSegmentFile(), readCommittedScope, segment)); } else if ((SegmentStatus.LOAD_FAILURE == segment.getSegmentStatus() || SegmentStatus.COMPACTED == segment.getSegmentStatus() || SegmentStatus.MARKED_FOR_DELETE == segment.getSegmentStatus())) { listOfInvalidSegments.add(new Segment(segment.getLoadName(), segment.getSegmentFile())); } else if (SegmentStatus.INSERT_IN_PROGRESS == segment.getSegmentStatus() || SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == segment.getSegmentStatus()) { listOfInProgressSegments.add( new Segment(segment.getLoadName(), segment.getSegmentFile(), readCommittedScope));
/** * Returns the invalid timestamp range of a segment. * @return */ public List<UpdateVO> getInvalidTimestampRange() { List<UpdateVO> ranges = new ArrayList<UpdateVO>(); for (LoadMetadataDetails segment : segmentDetails) { if ((SegmentStatus.LOAD_FAILURE == segment.getSegmentStatus() || SegmentStatus.COMPACTED == segment.getSegmentStatus() || SegmentStatus.MARKED_FOR_DELETE == segment.getSegmentStatus())) { UpdateVO range = new UpdateVO(); range.setSegmentId(segment.getLoadName()); range.setFactTimestamp(segment.getLoadStartTime()); if (!segment.getUpdateDeltaStartTimestamp().isEmpty() && !segment.getUpdateDeltaEndTimestamp().isEmpty()) { range.setUpdateDeltaStartTimestamp( CarbonUpdateUtil.getTimeStampAsLong(segment.getUpdateDeltaStartTimestamp())); range.setLatestUpdateTimestamp( CarbonUpdateUtil.getTimeStampAsLong(segment.getUpdateDeltaEndTimestamp())); } ranges.add(range); } } return ranges; }
if ((segment.getSegmentStatus() == SegmentStatus.SUCCESS || segment.getSegmentStatus() == SegmentStatus.LOAD_PARTIAL_SUCCESS) && segment.getSegmentFile() != null) { List<String> toBeDeletedIndexFiles = new ArrayList<>();
if (segment.getSegmentStatus() == SegmentStatus.STREAMING || segment.getSegmentStatus() == SegmentStatus.STREAMING_FINISH) { continue;
Integer result = compareDateValues(loadMetadata.getLoadStartTimeAsLong(), loadStartTime); if (result < 0) { segmentStatus = loadMetadata.getSegmentStatus(); if (SegmentStatus.COMPACTED == segmentStatus) { LOG.info("Ignoring the segment : " + loadMetadata.getLoadName()
/** * Return true if insert overwrite is in progress for specified table */ public static Boolean isOverwriteInProgressInTable(CarbonTable carbonTable) { if (carbonTable == null) { return false; } boolean loadInProgress = false; String metaPath = carbonTable.getMetadataPath(); LoadMetadataDetails[] listOfLoadFolderDetailsArray = SegmentStatusManager.readLoadMetadata(metaPath); if (listOfLoadFolderDetailsArray.length != 0) { for (LoadMetadataDetails loaddetail :listOfLoadFolderDetailsArray) { SegmentStatus segmentStatus = loaddetail.getSegmentStatus(); if (segmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS) { loadInProgress = isLoadInProgress(carbonTable.getAbsoluteTableIdentifier(), loaddetail.getLoadName()); } } } return loadInProgress; }
/** * Return true if any load or insert overwrite is in progress for specified table */ public static Boolean isLoadInProgressInTable(CarbonTable carbonTable) { if (carbonTable == null) { return false; } boolean loadInProgress = false; String metaPath = carbonTable.getMetadataPath(); LoadMetadataDetails[] listOfLoadFolderDetailsArray = SegmentStatusManager.readLoadMetadata(metaPath); if (listOfLoadFolderDetailsArray.length != 0) { for (LoadMetadataDetails loaddetail :listOfLoadFolderDetailsArray) { SegmentStatus segmentStatus = loaddetail.getSegmentStatus(); if (segmentStatus == SegmentStatus.INSERT_IN_PROGRESS || segmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS) { loadInProgress = isLoadInProgress(carbonTable.getAbsoluteTableIdentifier(), loaddetail.getLoadName()); } } } return loadInProgress; }
CarbonTablePath.addSegmentPrefix(oneLoad.getLoadName()) + LockUsage.LOCK); try { if (oneLoad.getSegmentStatus() == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS || oneLoad.getSegmentStatus() == SegmentStatus.INSERT_IN_PROGRESS) { if (segmentLock.lockWithRetries(1, 5)) { LOGGER.info("Info: Acquired segment lock on segment:" + oneLoad.getLoadName());
SegmentStatus loadStatus = loadMetadataDetail.getSegmentStatus(); if (loadStatus == SegmentStatus.SUCCESS || loadStatus == SegmentStatus.LOAD_PARTIAL_SUCCESS) {
boolean updated = false; for (LoadMetadataDetails detail : details) { if (SegmentStatus.STREAMING == detail.getSegmentStatus()) { detail.setLoadEndTime(System.currentTimeMillis()); detail.setSegmentStatus(SegmentStatus.STREAMING_FINISH);
for (LoadMetadataDetails detail : details) { if (FileFormat.ROW_V1 == detail.getFileFormat()) { if (SegmentStatus.STREAMING == detail.getSegmentStatus()) { streamSegment = detail; break;
if (!metadataDetail.getSegmentStatus().equals(SegmentStatus.SUCCESS)) { String readPath = CarbonTablePath.getSegmentFilesLocation(loadModel.getTablePath()) + CarbonCommonConstants.FILE_SEPARATOR + segmentFileName