/** * a new constructor with carbon index files * * @param indexFiles carbon index files */ public LatestFilesReadCommittedScope(CarbonFile[] indexFiles, Configuration configuration) { this.configuration = configuration; takeCarbonIndexFileSnapShot(indexFiles); }
/** * * @return map of Absolute path of index file as key and null as value -- without mergeIndex * map of AbsolutePath with fileName of MergeIndex parent file as key and mergeIndexFileName * as value -- with mergeIndex * @throws IOException */ public Map<String, String> getCommittedIndexFile() throws IOException { return readCommittedScope.getCommittedIndexFile(this); }
public TableStatusReadCommittedScope(AbsoluteTableIdentifier identifier, Configuration configuration) throws IOException { this.identifier = identifier; this.configuration = configuration; takeCarbonIndexFileSnapShot(); }
public ReadCommittedScope getReadCommitted(JobContext job, AbsoluteTableIdentifier identifier) throws IOException { if (readCommittedScope == null) { ReadCommittedScope readCommittedScope; if (job.getConfiguration().getBoolean(CARBON_TRANSACTIONAL_TABLE, true)) { readCommittedScope = new TableStatusReadCommittedScope(identifier, job.getConfiguration()); } else { readCommittedScope = getReadCommittedScope(job.getConfiguration()); if (readCommittedScope == null) { readCommittedScope = new LatestFilesReadCommittedScope(identifier.getTablePath(), job.getConfiguration()); } } this.readCommittedScope = readCommittedScope; } return readCommittedScope; } }
if (uniqueSegments.get(segmentId) == null) { segments.add(Segment.toSegment(segmentId, new TableStatusReadCommittedScope(table.getAbsoluteTableIdentifier(), loadMetadataDetails, FileFactory.getConfiguration()))); uniqueSegments.put(segmentId, 1);
LoadMetadataDetails[] loadMetadataDetails = readCommittedScope.getSegmentList(); readCommittedScope.getConfiguration()); SegmentStatusManager.ValidAndInvalidSegmentsInfo segments = segmentStatusManager .getValidAndInvalidSegments(loadMetadataDetails, this.readCommittedScope);
String segId = getSegmentID(carbonIndexFiles[i].getName(), indexFilePath); new ReadCommittedIndexFileSnapShot(indexFileStore, segmentTimestampUpdaterMap); this.readCommittedIndexFileSnapShot = readCommittedIndexFileSnapShot; prepareLoadMetadata();
readCommittedScope = new LatestFilesReadCommittedScope( identifier.getTablePath() + "/Fact/Part0/Segment_null/", job.getConfiguration()); } else { readCommittedScope = getReadCommittedScope(job.getConfiguration()); if (readCommittedScope == null) { readCommittedScope = new LatestFilesReadCommittedScope(identifier.getTablePath(), job .getConfiguration()); } else { readCommittedScope.setConfiguration(job.getConfiguration()); externalTableSegments.add(seg); } else { LoadMetadataDetails[] loadMetadataDetails = readCommittedScope.getSegmentList(); for (LoadMetadataDetails load : loadMetadataDetails) { seg = new Segment(load.getLoadName(), null, readCommittedScope);
public Configuration getConfiguration() { return readCommittedScope.getConfiguration(); }
@Override public Map<String, String> getCommittedIndexFile(Segment segment) throws IOException { Map<String, String> indexFileStore = new HashMap<>(); Map<String, List<String>> snapShot = readCommittedIndexFileSnapShot.getSegmentIndexFileMap(); String segName; if (segment.getSegmentNo() != null) { segName = segment.getSegmentNo(); } else { segName = segment.getSegmentFileName(); } List<String> index = snapShot.get(segName); if (null == index) { index = new LinkedList<>(); } for (String indexPath : index) { if (indexPath.endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) { indexFileStore.put(indexPath, indexPath.substring(indexPath.lastIndexOf('/') + 1)); } else { indexFileStore.put(indexPath, null); } } return indexFileStore; }
public SegmentRefreshInfo getSegmentRefreshInfo(UpdateVO updateVo) throws IOException { return readCommittedScope.getCommittedSegmentRefreshInfo(this, updateVo); }
@Override public SegmentRefreshInfo getCommittedSegmentRefreshInfo(Segment segment, UpdateVO updateVo) throws IOException { Map<String, SegmentRefreshInfo> snapShot = readCommittedIndexFileSnapShot.getSegmentTimestampUpdaterMap(); String segName; if (segment.getSegmentNo() != null) { segName = segment.getSegmentNo(); } else { segName = segment.getSegmentFileName(); } SegmentRefreshInfo segmentRefreshInfo = snapShot.get(segName); return segmentRefreshInfo; }
readCommittedScope = new TableStatusReadCommittedScope(identifier, loadMetadataDetails, configuration);
LoadMetadataDetails[] loadMetadataDetails = readCommittedScope.getSegmentList(); new SegmentStatusManager(identifier, readCommittedScope.getConfiguration()) .getValidAndInvalidSegments(loadMetadataDetails, readCommittedScope); Map<String, Long> blockRowCountMapping = new HashMap<>();
/** * a new constructor of this class * * @param path carbon file path * @param segmentId segment id */ public LatestFilesReadCommittedScope(String path, String segmentId, Configuration configuration) throws IOException { this.configuration = configuration; Objects.requireNonNull(path); this.carbonFilePath = path; this.segmentId = segmentId; takeCarbonIndexFileSnapShot(); }
@Override public LoadMetadataDetails[] getSegmentList() throws IOException { try { if (loadMetadataDetails == null) { takeCarbonIndexFileSnapShot(); } return loadMetadataDetails; } catch (IOException ex) { throw new IOException("Problem encountered while reading the Table Status file.", ex); } }
private void prepareLoadMetadata() { int loadCount = 0; Map<String, List<String>> snapshotMap = this.readCommittedIndexFileSnapShot.getSegmentIndexFileMap(); LoadMetadataDetails[] loadMetadataDetailsArray = new LoadMetadataDetails[snapshotMap.size()]; String segmentID; for (Map.Entry<String, List<String>> entry : snapshotMap.entrySet()) { segmentID = entry.getKey(); LoadMetadataDetails loadMetadataDetails = new LoadMetadataDetails(); long timeSet; try { timeSet = Long.parseLong(segmentID); } catch (NumberFormatException nu) { timeSet = 0; } loadMetadataDetails.setLoadEndTime(timeSet); loadMetadataDetails.setLoadStartTime(timeSet); loadMetadataDetails.setSegmentStatus(SegmentStatus.SUCCESS); loadMetadataDetails.setLoadName(segmentID); loadMetadataDetailsArray[loadCount++] = loadMetadataDetails; } this.loadMetadataDetails = loadMetadataDetailsArray; }
@Override public LoadMetadataDetails[] getSegmentList() throws IOException { try { if (loadMetadataDetails == null) { takeCarbonIndexFileSnapShot(); } return loadMetadataDetails; } catch (IOException ex) { throw new IOException("Problem encountered while reading the Table Status file.", ex); } }
@Override public void takeCarbonIndexFileSnapShot() throws IOException { // Read the current file Path get the list of indexes from the path. CarbonFile file = FileFactory.getCarbonFile(carbonFilePath, configuration); CarbonFile[] carbonIndexFiles = null; if (file.isDirectory()) { if (segmentId == null) { List<CarbonFile> indexFiles = new ArrayList<>(); SegmentIndexFileStore.getCarbonIndexFilesRecursively(file, indexFiles); carbonIndexFiles = indexFiles.toArray(new CarbonFile[0]); } else { String segmentPath = CarbonTablePath.getSegmentPath(carbonFilePath, segmentId); carbonIndexFiles = SegmentIndexFileStore.getCarbonIndexFiles(segmentPath, configuration); } if (carbonIndexFiles.length == 0) { throw new IOException( "No Index files are present in the table location :" + carbonFilePath); } takeCarbonIndexFileSnapShot(carbonIndexFiles); } else { throw new IOException("Path is not pointing to directory"); } }