/** * * @param segID * @param actualBlockName * @return null if block is not present in segment update status. */ private SegmentUpdateDetails getDetailsForABlock(String segID, String actualBlockName) { String blockIdentifier = CarbonUpdateUtil .getSegmentBlockNameKey(segID, actualBlockName); return blockAndDetailsMap.get(blockIdentifier); }
/** * This method returns the list of Blocks associated with the segment * from the SegmentUpdateDetails List. * @param segmentName * @return */ public List<String> getBlockNameFromSegment(String segmentName) { List<String> blockNames = new ArrayList<String>(); for (SegmentUpdateDetails block : updateDetails) { if (block.getSegmentName().equalsIgnoreCase(segmentName) && !CarbonUpdateUtil .isBlockInvalid(block.getSegmentStatus())) { blockNames.add(block.getBlockName()); } } return blockNames; }
/** * This will tell whether the max query timeout has been expired or not. * @param fileTimestamp * @return */ public static boolean isMaxQueryTimeoutExceeded(long fileTimestamp) { // record current time. long currentTime = CarbonUpdateUtil.readCurrentTime(); int maxTime; try { maxTime = Integer.parseInt(CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.MAX_QUERY_EXECUTION_TIME)); } catch (NumberFormatException e) { maxTime = CarbonCommonConstants.DEFAULT_MAX_QUERY_EXECUTION_TIME; } long difference = currentTime - fileTimestamp; long minutesElapsed = (difference / (1000 * 60)); return minutesElapsed > maxTime; }
if (deleteStaleCarbonDataFiles(segment, allSegmentFiles, updateStatusManager)) { updateSegmentFile = true; compareTimestampsAndDelete(invalidFile, forceDelete, false); if (compareTimestampsAndDelete(invalidFile, forceDelete, false)) { updateSegmentFile = true; for (CarbonFile invalidFile : invalidDeleteDeltaFiles) { boolean doForceDelete = true; compareTimestampsAndDelete(invalidFile, doForceDelete, false); if (CarbonUpdateUtil.isBlockInvalid(block.getSegmentStatus())) { completeListOfDeleteDeltaFiles = updateStatusManager .getDeleteDeltaInvalidFilesList(block, true, for (CarbonFile invalidFile : completeListOfDeleteDeltaFiles) { compareTimestampsAndDelete(invalidFile, forceDelete, false); compareTimestampsAndDelete(invalidFile, forceDelete, false); updateTableMetadataStatus( new HashSet<Segment>(segmentFilesToBeUpdated), table, compareTimestampsAndDelete(invalidFile, forceDelete, true);
String key = CarbonUpdateUtil.getSegmentBlockNameKey(segmentId, blockName); if (null == details || !CarbonUpdateUtil.isBlockInvalid(details.getSegmentStatus())) { Long blockCount = blockRowCountMapping.get(key); if (blockCount == null) {
/** * Update table status * @param updatedSegmentsList * @param table * @param updatedTimeStamp * @param isTimestampUpdationRequired * @param segmentsToBeDeleted * @return */ public static boolean updateTableMetadataStatus(Set<Segment> updatedSegmentsList, CarbonTable table, String updatedTimeStamp, boolean isTimestampUpdationRequired, List<Segment> segmentsToBeDeleted) { return updateTableMetadataStatus(updatedSegmentsList, table, updatedTimeStamp, isTimestampUpdationRequired, segmentsToBeDeleted, new ArrayList<Segment>(), ""); }
CarbonTablePath.getMetadataPath(identifier.getTablePath()) + CarbonCommonConstants.FILE_SEPARATOR + CarbonUpdateUtil.getUpdateStatusFileName(updateStatusFileIdentifier);
final Long endTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(endTimeStamp); final Long startTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(startTimeStamp); final long factTimeStampFinal = factTimeStamp; && CarbonUpdateUtil.isBlockInvalid(blockDetails.getSegmentStatus())) { validBlock = false;
CarbonUpdateUtil.updateSegmentStatus(segmentUpdateDetails, table, timestamp, true); if (loadMetadata.getLoadName().equalsIgnoreCase("0")) { loadMetadata.setUpdateStatusFileName( CarbonUpdateUtil.getUpdateStatusFileName(timestamp));
public DeleteDeltaInfo(String[] deleteDeltaFile) { this.deleteDeltaFile = deleteDeltaFile; this.latestDeleteDeltaFileTimestamp = CarbonUpdateUtil.getLatestDeleteDeltaTimestamp(deleteDeltaFile); }
public boolean addBlocklet(String blockletId, String offset, Integer pageId) throws Exception { DeleteDeltaBlockletDetails blocklet = new DeleteDeltaBlockletDetails(blockletId, pageId); try { blocklet.addDeletedRow(CarbonUpdateUtil.getIntegerValue(offset)); return addBlockletDetails(blocklet); } catch (Exception e) { if (LOGGER.isDebugEnabled()) { LOGGER.debug(e.getMessage()); } throw e; } } }
compareTimestampsAndDelete(invalidFile, true, false); if (compareTimestampsAndDelete(invalidFile, true, false)) { updateSegmentFile = true;
protected void setDataFileAttributesInModel(CarbonLoadModel loadModel, CompactionType compactionType, CarbonFactDataHandlerModel carbonFactDataHandlerModel) throws IOException { CarbonDataFileAttributes carbonDataFileAttributes; if (compactionType == CompactionType.IUD_UPDDEL_DELTA) { long taskNo = CarbonUpdateUtil.getLatestTaskIdForSegment(loadModel.getSegment(), loadModel.getTablePath()); // Increase the Task Index as in IUD_UPDDEL_DELTA_COMPACTION the new file will // be written in same segment. So the TaskNo should be incremented by 1 from max val. long index = taskNo + 1; carbonDataFileAttributes = new CarbonDataFileAttributes(index, loadModel.getFactTimeStamp()); } else { carbonDataFileAttributes = new CarbonDataFileAttributes(Long.parseLong(loadModel.getTaskNo()), loadModel.getFactTimeStamp()); } carbonFactDataHandlerModel.setCarbonDataFileAttributes(carbonDataFileAttributes); }
/** * Update the table status file with the dropped partitions information * * @param carbonTable * @param uniqueId * @param toBeUpdatedSegments * @param toBeDeleteSegments * @throws IOException */ public static void commitDropPartitions(CarbonTable carbonTable, String uniqueId, List<String> toBeUpdatedSegments, List<String> toBeDeleteSegments, String uuid) throws IOException { if (toBeDeleteSegments.size() > 0 || toBeUpdatedSegments.size() > 0) { Set<Segment> segmentSet = new HashSet<>( new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier()) .getValidAndInvalidSegments().getValidSegments()); CarbonUpdateUtil.updateTableMetadataStatus(segmentSet, carbonTable, uniqueId, true, Segment.toSegmentList(toBeDeleteSegments, null), Segment.toSegmentList(toBeUpdatedSegments, null), uuid); } }
if (loadDetail.getLoadName().equalsIgnoreCase("0")) { loadDetail .setUpdateStatusFileName(CarbonUpdateUtil.getUpdateStatusFileName(timestamp));
/** * updates segment status and modificaton time details * * @param loadMetadata */ public static void updateSegmentMetadataDetails(LoadMetadataDetails loadMetadata) { // update status only if the segment is not marked for delete if (SegmentStatus.MARKED_FOR_DELETE != loadMetadata.getSegmentStatus()) { loadMetadata.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE); loadMetadata.setModificationOrdeletionTimesStamp(CarbonUpdateUtil.readCurrentTime()); } }
/** * check the block whether is valid * * @param segName segment name * @param blockName block name * @return the status of block whether is valid */ public boolean isBlockValid(String segName, String blockName) { SegmentUpdateDetails details = getDetailsForABlock(segName, blockName); return details == null || !CarbonUpdateUtil.isBlockInvalid(details.getSegmentStatus()); } /**
context.getConfiguration()).getValidAndInvalidSegments().getValidSegments()); if (updateTime != null) { CarbonUpdateUtil.updateTableMetadataStatus(segmentSet, carbonTable, updateTime, true, segmentDeleteList); } else if (uniqueId != null) { CarbonUpdateUtil.updateTableMetadataStatus(segmentSet, carbonTable, uniqueId, true, segmentDeleteList);
CarbonUpdateUtil.getUpdateStatusFileName(updatedTimeStamp));
/** * populate the block and its details in a map. */ private void populateMap() { blockAndDetailsMap = new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); for (SegmentUpdateDetails blockDetails : updateDetails) { String blockIdentifier = CarbonUpdateUtil .getSegmentBlockNameKey(blockDetails.getSegmentName(), blockDetails.getActualBlockName()); blockAndDetailsMap.put(blockIdentifier, blockDetails); } }