synchronized SegmentInfos toLiveInfos(SegmentInfos sis) { final SegmentInfos newSIS = new SegmentInfos(sis.getIndexCreatedVersionMajor()); final Map<SegmentCommitInfo,SegmentCommitInfo> liveSIS = new HashMap<>(); for(SegmentCommitInfo info : segmentInfos) { liveSIS.put(info, info); } for(SegmentCommitInfo info : sis) { SegmentCommitInfo liveInfo = liveSIS.get(info); if (liveInfo != null) { info = liveInfo; } newSIS.add(info); } return newSIS; }
/** * Check index version in given directory. It assumes that that all commits * in the Lucene segment file were done with the same version. * * @param dir directory with index * @thows IOException if the directory cannot be opened */ private static void checkDir(File dir) throws Exception { LockFactory lockfact = NativeFSLockFactory.INSTANCE; int segVersion; try (Directory indexDirectory = FSDirectory.open(dir.toPath(), lockfact)) { SegmentInfos segInfos = null; try { segInfos = SegmentInfos.readLatestCommit(indexDirectory); segVersion = segInfos.getIndexCreatedVersionMajor(); } catch (IndexNotFoundException e) { return; } } if (segVersion != Version.LATEST.major) { throw new IndexVersionException( String.format("Directory %s has index of version %d and Lucene has %d", dir.toString(), segVersion, Version.LATEST.major)); } } }
private void validateMergeReader(CodecReader leaf) { LeafMetaData segmentMeta = leaf.getMetaData(); if (segmentInfos.getIndexCreatedVersionMajor() != segmentMeta.getCreatedVersionMajor()) { throw new IllegalArgumentException("Cannot merge a segment that has been created with major version " + segmentMeta.getCreatedVersionMajor() + " into this index which has been created by major version " + segmentInfos.getIndexCreatedVersionMajor()); } if (segmentInfos.getIndexCreatedVersionMajor() >= 7 && segmentMeta.getMinVersion() == null) { throw new IllegalStateException("Indexes created on or after Lucene 7 must record the created version major, but " + leaf + " hides it"); } Sort leafIndexSort = segmentMeta.getSort(); if (config.getIndexSort() != null && leafIndexSort != null && config.getIndexSort().equals(leafIndexSort) == false) { throw new IllegalArgumentException("cannot change index sort from " + leafIndexSort + " to " + config.getIndexSort()); } }
/** * Obtain a ReadersAndLiveDocs instance from the * readerPool. If create is true, you must later call * {@link #release(ReadersAndUpdates, boolean)}. */ synchronized ReadersAndUpdates get(SegmentCommitInfo info, boolean create) { assert info.info.dir == originalDirectory: "info.dir=" + info.info.dir + " vs " + originalDirectory; if (closed.get()) { assert readerMap.isEmpty() : "Reader map is not empty: " + readerMap; throw new AlreadyClosedException("ReaderPool is already closed"); } ReadersAndUpdates rld = readerMap.get(info); if (rld == null) { if (create == false) { return null; } rld = new ReadersAndUpdates(segmentInfos.getIndexCreatedVersionMajor(), info, newPendingDeletes(info)); // Steal initial reference: readerMap.put(info, rld); } else { assert rld.info == info: "rld.info=" + rld.info + " info=" + info + " isLive?=" + assertInfoIsLive(rld.info) + " vs " + assertInfoIsLive(info); } if (create) { // Return ref to caller: rld.incRef(); } assert noDups(); return rld; }
ReaderPool(Directory directory, Directory originalDirectory, SegmentInfos segmentInfos, FieldInfos.FieldNumbers fieldNumbers, LongSupplier completedDelGenSupplier, InfoStream infoStream, String softDeletesField, StandardDirectoryReader reader) throws IOException { this.directory = directory; this.originalDirectory = originalDirectory; this.segmentInfos = segmentInfos; this.fieldNumbers = fieldNumbers; this.completedDelGenSupplier = completedDelGenSupplier; this.infoStream = infoStream; this.softDeletesField = softDeletesField; if (reader != null) { // Pre-enroll all segment readers into the reader pool; this is necessary so // any in-memory NRT live docs are correctly carried over, and so NRT readers // pulled from this IW share the same segment reader: List<LeafReaderContext> leaves = reader.leaves(); assert segmentInfos.size() == leaves.size(); for (int i=0;i<leaves.size();i++) { LeafReaderContext leaf = leaves.get(i); SegmentReader segReader = (SegmentReader) leaf.reader(); SegmentReader newReader = new SegmentReader(segmentInfos.info(i), segReader, segReader.getLiveDocs(), segReader.getHardLiveDocs(), segReader.numDocs(), true); readerMap.put(newReader.getOriginalSegmentInfo(), new ReadersAndUpdates(segmentInfos.getIndexCreatedVersionMajor(), newReader, newPendingDeletes(newReader, newReader.getOriginalSegmentInfo()))); } } }
if (oldReader == null || commitInfo.info.getUseCompoundFile() != oldReader.getSegmentInfo().info.getUseCompoundFile()) { newReader = new SegmentReader(commitInfo, infos.getIndexCreatedVersionMajor(), IOContext.READ); newReaders[i] = newReader; } else {
if (segmentInfos.getIndexCreatedVersionMajor() != sis.getIndexCreatedVersionMajor()) { throw new IllegalArgumentException("Cannot use addIndexes(Directory) with indexes that have been created " + "by a different Lucene version. The current index was generated by Lucene " + segmentInfos.getIndexCreatedVersionMajor() + " while one of the directories contains an index that was generated with Lucene " + sis.getIndexCreatedVersionMajor());
if (infoStream != null) infoStream.print(" test: open reader........."); reader = new SegmentReader(info, sis.getIndexCreatedVersionMajor(), IOContext.DEFAULT); msg(infoStream, String.format(Locale.ROOT, "OK [took %.3f sec]", nsToSec(System.nanoTime()-startOpenReaderNS)));
docWriter = new DocumentsWriter(flushNotifications, segmentInfos.getIndexCreatedVersionMajor(), pendingNumDocs, enableTestPoints, this::newSegmentName, config, directoryOrig, directory, globalFieldNumberMap);
final int luceneIndexCreatedVersionMajor = Lucene.readSegmentInfos(sources[0]).getIndexCreatedVersionMajor(); new SegmentInfos(luceneIndexCreatedVersionMajor).commit(target);
synchronized SegmentInfos toLiveInfos(SegmentInfos sis) { final SegmentInfos newSIS = new SegmentInfos(sis.getIndexCreatedVersionMajor()); final Map<SegmentCommitInfo,SegmentCommitInfo> liveSIS = new HashMap<>(); for(SegmentCommitInfo info : segmentInfos) { liveSIS.put(info, info); } for(SegmentCommitInfo info : sis) { SegmentCommitInfo liveInfo = liveSIS.get(info); if (liveInfo != null) { info = liveInfo; } newSIS.add(info); } return newSIS; }
private void validateMergeReader(CodecReader leaf) { LeafMetaData segmentMeta = leaf.getMetaData(); if (segmentInfos.getIndexCreatedVersionMajor() != segmentMeta.getCreatedVersionMajor()) { throw new IllegalArgumentException("Cannot merge a segment that has been created with major version " + segmentMeta.getCreatedVersionMajor() + " into this index which has been created by major version " + segmentInfos.getIndexCreatedVersionMajor()); } if (segmentInfos.getIndexCreatedVersionMajor() >= 7 && segmentMeta.getMinVersion() == null) { throw new IllegalStateException("Indexes created on or after Lucene 7 must record the created version major, but " + leaf + " hides it"); } Sort leafIndexSort = segmentMeta.getSort(); if (config.getIndexSort() != null && leafIndexSort != null && config.getIndexSort().equals(leafIndexSort) == false) { throw new IllegalArgumentException("cannot change index sort from " + leafIndexSort + " to " + config.getIndexSort()); } }
int createdMajor = SegmentInfos.readLatestCommit(srcDir).getIndexCreatedVersionMajor(); new SegmentInfos(createdMajor).commit(destDir); IndexWriter writer = new IndexWriter(destDir, new IndexWriterConfig());
/** * Obtain a ReadersAndLiveDocs instance from the * readerPool. If create is true, you must later call * {@link #release(ReadersAndUpdates, boolean)}. */ synchronized ReadersAndUpdates get(SegmentCommitInfo info, boolean create) { assert info.info.dir == originalDirectory: "info.dir=" + info.info.dir + " vs " + originalDirectory; if (closed.get()) { assert readerMap.isEmpty() : "Reader map is not empty: " + readerMap; throw new AlreadyClosedException("ReaderPool is already closed"); } ReadersAndUpdates rld = readerMap.get(info); if (rld == null) { if (create == false) { return null; } rld = new ReadersAndUpdates(segmentInfos.getIndexCreatedVersionMajor(), info, newPendingDeletes(info)); // Steal initial reference: readerMap.put(info, rld); } else { assert rld.info == info: "rld.info=" + rld.info + " info=" + info + " isLive?=" + assertInfoIsLive(rld.info) + " vs " + assertInfoIsLive(info); } if (create) { // Return ref to caller: rld.incRef(); } assert noDups(); return rld; }
final int luceneIndexCreatedVersionMajor = Lucene.readSegmentInfos(sources[0]).getIndexCreatedVersionMajor(); new SegmentInfos(luceneIndexCreatedVersionMajor).commit(target);
final int luceneIndexCreatedVersionMajor = Lucene.readSegmentInfos(sources[0]).getIndexCreatedVersionMajor(); new SegmentInfos(luceneIndexCreatedVersionMajor).commit(target);
ReaderPool(Directory directory, Directory originalDirectory, SegmentInfos segmentInfos, FieldInfos.FieldNumbers fieldNumbers, LongSupplier completedDelGenSupplier, InfoStream infoStream, String softDeletesField, StandardDirectoryReader reader) throws IOException { this.directory = directory; this.originalDirectory = originalDirectory; this.segmentInfos = segmentInfos; this.fieldNumbers = fieldNumbers; this.completedDelGenSupplier = completedDelGenSupplier; this.infoStream = infoStream; this.softDeletesField = softDeletesField; if (reader != null) { // Pre-enroll all segment readers into the reader pool; this is necessary so // any in-memory NRT live docs are correctly carried over, and so NRT readers // pulled from this IW share the same segment reader: List<LeafReaderContext> leaves = reader.leaves(); assert segmentInfos.size() == leaves.size(); for (int i=0;i<leaves.size();i++) { LeafReaderContext leaf = leaves.get(i); SegmentReader segReader = (SegmentReader) leaf.reader(); SegmentReader newReader = new SegmentReader(segmentInfos.info(i), segReader, segReader.getLiveDocs(), segReader.getHardLiveDocs(), segReader.numDocs(), true); readerMap.put(newReader.getOriginalSegmentInfo(), new ReadersAndUpdates(segmentInfos.getIndexCreatedVersionMajor(), newReader, newPendingDeletes(newReader, newReader.getOriginalSegmentInfo()))); } } }
public void split(Path destDir, String[] segs) throws IOException { Files.createDirectories(destDir); FSDirectory destFSDir = FSDirectory.open(destDir); SegmentInfos destInfos = new SegmentInfos(infos.getIndexCreatedVersionMajor()); destInfos.counter = infos.counter; for (String n : segs) { SegmentCommitInfo infoPerCommit = getInfo(n); SegmentInfo info = infoPerCommit.info; // Same info just changing the dir: SegmentInfo newInfo = new SegmentInfo(destFSDir, info.getVersion(), info.getMinVersion(), info.name, info.maxDoc(), info.getUseCompoundFile(), info.getCodec(), info.getDiagnostics(), info.getId(), new HashMap<>(), null); destInfos.add(new SegmentCommitInfo(newInfo, infoPerCommit.getDelCount(), infoPerCommit.getSoftDelCount(), infoPerCommit.getDelGen(), infoPerCommit.getFieldInfosGen(), infoPerCommit.getDocValuesGen())); // now copy files over Collection<String> files = infoPerCommit.files(); for (final String srcName : files) { Path srcFile = dir.resolve(srcName); Path destFile = destDir.resolve(srcName); Files.copy(srcFile, destFile); } } destInfos.changed(); destInfos.commit(destFSDir); // System.out.println("destDir:"+destDir.getAbsolutePath()); } }
if (oldReader == null || commitInfo.info.getUseCompoundFile() != oldReader.getSegmentInfo().info.getUseCompoundFile()) { newReader = new SegmentReader(commitInfo, infos.getIndexCreatedVersionMajor(), IOContext.READ); newReaders[i] = newReader; } else {
if (segmentInfos.getIndexCreatedVersionMajor() != sis.getIndexCreatedVersionMajor()) { throw new IllegalArgumentException("Cannot use addIndexes(Directory) with indexes that have been created " + "by a different Lucene version. The current index was generated by Lucene " + segmentInfos.getIndexCreatedVersionMajor() + " while one of the directories contains an index that was generated with Lucene " + sis.getIndexCreatedVersionMajor());