private static boolean isSegmentValid(LoadMetadataDetails seg) { return seg.getSegmentStatus() == SegmentStatus.SUCCESS || seg.getSegmentStatus() == SegmentStatus.LOAD_PARTIAL_SUCCESS || seg.getSegmentStatus() == SegmentStatus.MARKED_FOR_UPDATE; }
/** * @return the startLoadTime */ public long getLoadStartTime() { if (loadStartTime == null) { return CarbonCommonConstants.SEGMENT_LOAD_TIME_DEFAULT; } return convertTimeStampToLong(loadStartTime); }
/** * This method will create new segment id * * @param loadMetadataDetails * @return */ public static int createNewSegmentId(LoadMetadataDetails[] loadMetadataDetails) { int newSegmentId = getMaxSegmentId(loadMetadataDetails); newSegmentId++; return newSegmentId; }
/** * this method will delete the datamap folders during drop datamap * @throws MalformedDataMapCommandException */ private void deleteDatamap() throws MalformedDataMapCommandException { SegmentStatusManager ssm = new SegmentStatusManager(tableIdentifier); try { List<Segment> validSegments = ssm.getValidAndInvalidSegments().getValidSegments(); for (Segment segment : validSegments) { deleteDatamapData(segment); } } catch (IOException | RuntimeException ex) { throw new MalformedDataMapCommandException( "drop datamap failed, failed to delete datamap directory"); } }
private static SegmentStatusManager.ValidAndInvalidSegmentsInfo getValidAndInvalidSegments( CarbonTable carbonTable, Configuration configuration) throws IOException { SegmentStatusManager ssm = new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier(), configuration); return ssm.getValidAndInvalidSegments(); }
/** * updates segment status and modificaton time details * * @param loadMetadata */ public static void updateSegmentMetadataDetails(LoadMetadataDetails loadMetadata) { // update status only if the segment is not marked for delete if (SegmentStatus.MARKED_FOR_DELETE != loadMetadata.getSegmentStatus()) { loadMetadata.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE); loadMetadata.setModificationOrdeletionTimesStamp(CarbonUpdateUtil.readCurrentTime()); } }
private static boolean isLoadDeletionRequired(String metaDataLocation) { LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(metaDataLocation); if (details != null && details.length > 0) { for (LoadMetadataDetails oneRow : details) { if ((SegmentStatus.MARKED_FOR_DELETE == oneRow.getSegmentStatus() || SegmentStatus.COMPACTED == oneRow.getSegmentStatus() || SegmentStatus.INSERT_IN_PROGRESS == oneRow.getSegmentStatus() || SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == oneRow.getSegmentStatus()) && oneRow.getVisibility().equalsIgnoreCase("true")) { return true; } } } return false; }
TableSegmentRefresher(CarbonTable table) { SegmentUpdateStatusManager statusManager = new SegmentUpdateStatusManager(table); SegmentUpdateDetails[] updateStatusDetails = statusManager.getUpdateStatusDetails(); for (SegmentUpdateDetails updateDetails : updateStatusDetails) { UpdateVO updateVO = statusManager.getInvalidTimestampRange(updateDetails.getSegmentName()); segmentRefreshTime.put(updateVO.getSegmentId(), new SegmentRefreshInfo(updateVO.getCreatedOrUpdatedTimeStamp(), 0)); } }
private static LoadMetadataDetails getCurrentLoadStatusOfSegment(String segmentId, String metadataPath) { LoadMetadataDetails[] currentDetails = SegmentStatusManager.readLoadMetadata(metadataPath); for (LoadMetadataDetails oneLoad : currentDetails) { if (oneLoad.getLoadName().equalsIgnoreCase(segmentId)) { return oneLoad; } } return null; }
public ValidAndInvalidSegmentsInfo getValidAndInvalidSegments() throws IOException { return getValidAndInvalidSegments(null, null); }
private static boolean checkIfLoadCanBeDeletedPhysically(LoadMetadataDetails oneLoad, boolean isForceDelete) { if ((SegmentStatus.MARKED_FOR_DELETE == oneLoad.getSegmentStatus() || SegmentStatus.COMPACTED == oneLoad.getSegmentStatus())) { if (isForceDelete) { return true; } long deletionTime = oneLoad.getModificationOrdeletionTimesStamp(); return CarbonUpdateUtil.isMaxQueryTimeoutExceeded(deletionTime); } return false; }
@Override public int compare(LoadMetadataDetails seg1, LoadMetadataDetails seg2) { double seg1Id = Double.parseDouble(seg1.getLoadName()); double seg2Id = Double.parseDouble(seg2.getLoadName()); return Double.compare(seg1Id, seg2Id); } });
/** * @return updateStatusFileName */ private String getUpdatedStatusIdentifier() { if (segmentDetails.length == 0) { return null; } return segmentDetails[0].getUpdateStatusFileName(); }
/** * This method reads the load metadata file * * @param metadataFolderPath * @return */ public static LoadMetadataDetails[] readLoadMetadata(String metadataFolderPath) { String metadataFileName = metadataFolderPath + CarbonCommonConstants.FILE_SEPARATOR + CarbonTablePath.TABLE_STATUS_FILE; try { return readTableStatusFile(metadataFileName); } catch (IOException e) { return new LoadMetadataDetails[0]; } }
public static FileFormat getByOrdinal(int ordinal) { if (ordinal < 0 || ordinal >= FileFormat.values().length) { return COLUMNAR_V3; } switch (ordinal) { case 0: return COLUMNAR_V3; case 1: return ROW_V1; } return COLUMNAR_V3; } }
public SegmentRefreshInfo getCommittedSegmentRefreshInfo(Segment segment, UpdateVO updateVo) throws IOException { SegmentRefreshInfo segmentRefreshInfo; if (updateVo != null) { segmentRefreshInfo = new SegmentRefreshInfo(updateVo.getCreatedOrUpdatedTimeStamp(), 0); } else { segmentRefreshInfo = new SegmentRefreshInfo(0L, 0); } return segmentRefreshInfo; }
@Override public void deleteDatamapData() { SegmentStatusManager ssm = new SegmentStatusManager(getCarbonTable().getAbsoluteTableIdentifier()); try { List<Segment> validSegments = ssm.getValidAndInvalidSegments().getValidSegments(); for (Segment segment : validSegments) { deleteDatamapData(segment); } } catch (IOException e) { LOGGER.error("drop datamap failed, failed to delete datamap directory"); } }
public long getLoadEndTime() { if (timestamp == null) { return CarbonCommonConstants.SEGMENT_LOAD_TIME_DEFAULT; } return convertTimeStampToLong(timestamp); }
/** * This method reads the load history metadata file * * @param metadataFolderPath * @return */ public static LoadMetadataDetails[] readLoadHistoryMetadata(String metadataFolderPath) { String metadataFileName = metadataFolderPath + CarbonCommonConstants.FILE_SEPARATOR + CarbonTablePath.TABLE_STATUS_HISTORY_FILE; try { return readTableStatusFile(metadataFileName); } catch (IOException e) { return new LoadMetadataDetails[0]; } }
/** * @return the modificationOrdeletionTimesStamp */ public long getModificationOrdeletionTimesStamp() { if (null == modificationOrdeletionTimesStamp) { return 0; } return convertTimeStampToLong(modificationOrdeletionTimesStamp); }