/** * This will update the old table status details before clean files to the latest table status. * @param oldList * @param newList * @return */ public static List<LoadMetadataDetails> updateLoadMetadataFromOldToNew( LoadMetadataDetails[] oldList, LoadMetadataDetails[] newList) { List<LoadMetadataDetails> newListMetadata = new ArrayList<LoadMetadataDetails>(Arrays.asList(newList)); for (LoadMetadataDetails oldSegment : oldList) { if ("false".equalsIgnoreCase(oldSegment.getVisibility())) { newListMetadata.get(newListMetadata.indexOf(oldSegment)).setVisibility("false"); } } return newListMetadata; }
/** * Get the number of invisible segment info from segment info list. */ public static int countInvisibleSegments( LoadMetadataDetails[] segmentList, int maxSegmentId) { int invisibleSegmentCnt = 0; if (segmentList.length != 0) { for (LoadMetadataDetails eachSeg : segmentList) { // can not remove segment 0, there are some info will be used later // for example: updateStatusFileName // also can not remove the max segment id, // otherwise will impact the generation of segment id if (!eachSeg.getLoadName().equalsIgnoreCase("0") && !eachSeg.getLoadName().equalsIgnoreCase(String.valueOf(maxSegmentId)) && eachSeg.getVisibility().equalsIgnoreCase("false")) { invisibleSegmentCnt += 1; } } } return invisibleSegmentCnt; }
if (newSegment.getLoadName().equalsIgnoreCase("0") || newSegment.getLoadName().equalsIgnoreCase(String.valueOf(maxSegmentId))) { newSegment.setVisibility(oldSegment.getVisibility()); arrayOfVisibleSegments[visibleIdx] = newSegment; visibleIdx++; } else if ("false".equalsIgnoreCase(oldSegment.getVisibility())) { newSegment.setVisibility("false"); arrayOfInvisibleSegments[invisibleIdx] = newSegment;
private static boolean isLoadDeletionRequired(String metaDataLocation) { LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(metaDataLocation); if (details != null && details.length > 0) { for (LoadMetadataDetails oneRow : details) { if ((SegmentStatus.MARKED_FOR_DELETE == oneRow.getSegmentStatus() || SegmentStatus.COMPACTED == oneRow.getSegmentStatus() || SegmentStatus.INSERT_IN_PROGRESS == oneRow.getSegmentStatus() || SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == oneRow.getSegmentStatus()) && oneRow.getVisibility().equalsIgnoreCase("true")) { return true; } } } return false; }
private static boolean checkIfLoadCanBeDeleted(LoadMetadataDetails oneLoad, boolean isForceDelete) { if ((SegmentStatus.MARKED_FOR_DELETE == oneLoad.getSegmentStatus() || SegmentStatus.COMPACTED == oneLoad.getSegmentStatus() || SegmentStatus.INSERT_IN_PROGRESS == oneLoad.getSegmentStatus() || SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == oneLoad.getSegmentStatus()) && oneLoad.getVisibility().equalsIgnoreCase("true")) { if (isForceDelete) { return true; } long deletionTime = oneLoad.getModificationOrdeletionTimesStamp(); return CarbonUpdateUtil.isMaxQueryTimeoutExceeded(deletionTime); } return false; }