/** Confirms that the incoming index sort (if any) matches the existing index sort (if any). * This is unfortunately just best effort, because it could be the old index only has unsorted flushed segments built * before {@link Version#LUCENE_6_5_0} (flushed segments are sorted in Lucene 7.0). */ private void validateIndexSort() throws CorruptIndexException { Sort indexSort = config.getIndexSort(); if (indexSort != null) { for(SegmentCommitInfo info : segmentInfos) { Sort segmentIndexSort = info.info.getIndexSort(); if (segmentIndexSort != null && indexSort.equals(segmentIndexSort) == false) { throw new IllegalArgumentException("cannot change previous indexSort=" + segmentIndexSort + " (from segment=" + info + ") to new indexSort=" + indexSort); } else if (segmentIndexSort == null && info.info.getVersion().onOrAfter(Version.LUCENE_6_5_0)) { // Flushed segments are not sorted if they were built with a version prior to 6.5.0 throw new CorruptIndexException("segment not sorted with indexSort=" + segmentIndexSort, info.info.toString()); } } } }
private Sorter.DocMap maybeSortSegment(SegmentWriteState state) throws IOException { Sort indexSort = state.segmentInfo.getIndexSort(); if (indexSort == null) { return null; } List<Sorter.DocComparator> comparators = new ArrayList<>(); for (int i = 0; i < indexSort.getSort().length; i++) { SortField sortField = indexSort.getSort()[i]; PerField perField = getPerField(sortField.getField()); if (perField != null && perField.docValuesWriter != null && finishedDocValues.contains(perField.fieldInfo.name) == false) { perField.docValuesWriter.finish(state.segmentInfo.maxDoc()); Sorter.DocComparator cmp = perField.docValuesWriter.getDocComparator(state.segmentInfo.maxDoc(), sortField); comparators.add(cmp); finishedDocValues.add(perField.fieldInfo.name); } else { // safe to ignore, sort field with no values or already seen before } } Sorter sorter = new Sorter(indexSort); // returns null if the documents are already sorted return sorter.sort(state.segmentInfo.maxDoc(), comparators.toArray(new Sorter.DocComparator[comparators.size()])); }
SegmentMerger(List<CodecReader> readers, SegmentInfo segmentInfo, InfoStream infoStream, Directory dir, FieldInfos.FieldNumbers fieldNumbers, IOContext context) throws IOException { if (context.context != IOContext.Context.MERGE) { throw new IllegalArgumentException("IOContext.context should be MERGE; got: " + context.context); } mergeState = new MergeState(readers, segmentInfo, infoStream); directory = dir; this.codec = segmentInfo.getCodec(); this.context = context; this.fieldInfosBuilder = new FieldInfos.Builder(fieldNumbers); Version minVersion = Version.LATEST; for (CodecReader reader : readers) { Version leafMinVersion = reader.getMetaData().getMinVersion(); if (leafMinVersion == null) { minVersion = null; break; } if (minVersion.onOrAfter(leafMinVersion)) { minVersion = leafMinVersion; } } assert segmentInfo.minVersion == null : "The min version should be set by SegmentMerger for merged segments"; segmentInfo.minVersion = minVersion; if (mergeState.infoStream.isEnabled("SM")) { if (segmentInfo.getIndexSort() != null) { mergeState.infoStream.message("SM", "index sort during merge: " + segmentInfo.getIndexSort()); } } }
Sort indexSort = segmentInfo.getIndexSort(); if (indexSort == null) { return originalReaders;
public DefaultIndexingChain(DocumentsWriterPerThread docWriter) throws IOException { this.docWriter = docWriter; this.fieldInfos = docWriter.getFieldInfosBuilder(); this.docState = docWriter.docState; this.bytesUsed = docWriter.bytesUsed; final TermsHash termVectorsWriter; if (docWriter.getSegmentInfo().getIndexSort() == null) { storedFieldsConsumer = new StoredFieldsConsumer(docWriter); termVectorsWriter = new TermVectorsConsumer(docWriter); } else { storedFieldsConsumer = new SortingStoredFieldsConsumer(docWriter); termVectorsWriter = new SortingTermVectorsConsumer(docWriter); } termsHash = new FreqProxTermsWriter(docWriter, termVectorsWriter); }
if (docWriter.getSegmentInfo().getIndexSort() != null) { final Sort indexSort = docWriter.getSegmentInfo().getIndexSort(); validateIndexSortDVType(indexSort, fp.fieldInfo.name, dvType);
segInfoStat.compound = info.info.getUseCompoundFile(); msg(infoStream, " numFiles=" + info.files().size()); Sort indexSort = info.info.getIndexSort(); if (indexSort != null) { msg(infoStream, " sort=" + indexSort);
info.info.getDiagnostics(), info.info.getId(), info.info.getAttributes(), info.info.getIndexSort()); SegmentCommitInfo newInfoPerCommit = new SegmentCommitInfo(newInfo, info.getDelCount(), info.getSoftDelCount(), info.getDelGen(), info.getFieldInfosGen(), info.getDocValuesGen());
final Sort indexSort = segmentInfo.getIndexSort(); int numReaders = originalReaders.size(); leafDocMaps = new DocMap[numReaders];
output.writeMapOfStrings(si.getAttributes()); Sort indexSort = si.getIndexSort(); int numSortFields = indexSort == null ? 0 : indexSort.getSort().length; output.writeVInt(numSortFields);
assert !infos.contains(info): "dup info dir=" + info.info.dir + " name=" + info.info.name; Sort segmentIndexSort = info.info.getIndexSort();
this.si = si.clone(); this.originalSi = si; this.metaData = new LeafMetaData(createdVersionMajor, si.info.getMinVersion(), si.info.getIndexSort());
private void fillSegmentInfo(SegmentReader segmentReader, boolean verbose, boolean search, Map<String, Segment> segments) { SegmentCommitInfo info = segmentReader.getSegmentInfo(); assert segments.containsKey(info.info.name) == false; Segment segment = new Segment(info.info.name); segment.search = search; segment.docCount = segmentReader.numDocs(); segment.delDocCount = segmentReader.numDeletedDocs(); segment.version = info.info.getVersion(); segment.compound = info.info.getUseCompoundFile(); try { segment.sizeInBytes = info.sizeInBytes(); } catch (IOException e) { logger.trace(() -> new ParameterizedMessage("failed to get size for [{}]", info.info.name), e); } segment.memoryInBytes = segmentReader.ramBytesUsed(); segment.segmentSort = info.info.getIndexSort(); if (verbose) { segment.ramTree = Accountables.namedAccountable("root", segmentReader); } segment.attributes = info.info.getAttributes(); // TODO: add more fine grained mem stats values to per segment info here segments.put(info.info.name, segment); }
logger.trace(() -> new ParameterizedMessage("failed to get size for [{}]", info.info.name), e); segment.segmentSort = info.info.getIndexSort(); segment.attributes = info.info.getAttributes(); segments.put(info.info.name, segment);
/** Confirms that the incoming index sort (if any) matches the existing index sort (if any). * This is unfortunately just best effort, because it could be the old index only has unsorted flushed segments built * before {@link Version#LUCENE_6_5_0} (flushed segments are sorted in Lucene 7.0). */ private void validateIndexSort() throws CorruptIndexException { Sort indexSort = config.getIndexSort(); if (indexSort != null) { for(SegmentCommitInfo info : segmentInfos) { Sort segmentIndexSort = info.info.getIndexSort(); if (segmentIndexSort != null && indexSort.equals(segmentIndexSort) == false) { throw new IllegalArgumentException("cannot change previous indexSort=" + segmentIndexSort + " (from segment=" + info + ") to new indexSort=" + indexSort); } else if (segmentIndexSort == null && info.info.getVersion().onOrAfter(Version.LUCENE_6_5_0)) { // Flushed segments are not sorted if they were built with a version prior to 6.5.0 throw new CorruptIndexException("segment not sorted with indexSort=" + segmentIndexSort, info.info.toString()); } } } }
private Sorter.DocMap maybeSortSegment(SegmentWriteState state) throws IOException { Sort indexSort = state.segmentInfo.getIndexSort(); if (indexSort == null) { return null; } List<Sorter.DocComparator> comparators = new ArrayList<>(); for (int i = 0; i < indexSort.getSort().length; i++) { SortField sortField = indexSort.getSort()[i]; PerField perField = getPerField(sortField.getField()); if (perField != null && perField.docValuesWriter != null && finishedDocValues.contains(perField.fieldInfo.name) == false) { perField.docValuesWriter.finish(state.segmentInfo.maxDoc()); Sorter.DocComparator cmp = perField.docValuesWriter.getDocComparator(state.segmentInfo.maxDoc(), sortField); comparators.add(cmp); finishedDocValues.add(perField.fieldInfo.name); } else { // safe to ignore, sort field with no values or already seen before } } Sorter sorter = new Sorter(indexSort); // returns null if the documents are already sorted return sorter.sort(state.segmentInfo.maxDoc(), comparators.toArray(new Sorter.DocComparator[comparators.size()])); }
SegmentMerger(List<CodecReader> readers, SegmentInfo segmentInfo, InfoStream infoStream, Directory dir, FieldInfos.FieldNumbers fieldNumbers, IOContext context) throws IOException { if (context.context != IOContext.Context.MERGE) { throw new IllegalArgumentException("IOContext.context should be MERGE; got: " + context.context); } mergeState = new MergeState(readers, segmentInfo, infoStream); directory = dir; this.codec = segmentInfo.getCodec(); this.context = context; this.fieldInfosBuilder = new FieldInfos.Builder(fieldNumbers); Version minVersion = Version.LATEST; for (CodecReader reader : readers) { Version leafMinVersion = reader.getMetaData().getMinVersion(); if (leafMinVersion == null) { minVersion = null; break; } if (minVersion.onOrAfter(leafMinVersion)) { minVersion = leafMinVersion; } } assert segmentInfo.minVersion == null : "The min version should be set by SegmentMerger for merged segments"; segmentInfo.minVersion = minVersion; if (mergeState.infoStream.isEnabled("SM")) { if (segmentInfo.getIndexSort() != null) { mergeState.infoStream.message("SM", "index sort during merge: " + segmentInfo.getIndexSort()); } } }
private void fillSegmentInfo(SegmentReader segmentReader, boolean verbose, boolean search, Map<String, Segment> segments) { SegmentCommitInfo info = segmentReader.getSegmentInfo(); assert segments.containsKey(info.info.name) == false; Segment segment = new Segment(info.info.name); segment.search = search; segment.docCount = segmentReader.numDocs(); segment.delDocCount = segmentReader.numDeletedDocs(); segment.version = info.info.getVersion(); segment.compound = info.info.getUseCompoundFile(); try { segment.sizeInBytes = info.sizeInBytes(); } catch (IOException e) { logger.trace(() -> new ParameterizedMessage("failed to get size for [{}]", info.info.name), e); } segment.memoryInBytes = segmentReader.ramBytesUsed(); segment.segmentSort = info.info.getIndexSort(); if (verbose) { segment.ramTree = Accountables.namedAccountable("root", segmentReader); } segment.attributes = info.info.getAttributes(); // TODO: add more fine grained mem stats values to per segment info here segments.put(info.info.name, segment); }
public DefaultIndexingChain(DocumentsWriterPerThread docWriter) throws IOException { this.docWriter = docWriter; this.fieldInfos = docWriter.getFieldInfosBuilder(); this.docState = docWriter.docState; this.bytesUsed = docWriter.bytesUsed; final TermsHash termVectorsWriter; if (docWriter.getSegmentInfo().getIndexSort() == null) { storedFieldsConsumer = new StoredFieldsConsumer(docWriter); termVectorsWriter = new TermVectorsConsumer(docWriter); } else { storedFieldsConsumer = new SortingStoredFieldsConsumer(docWriter); termVectorsWriter = new SortingTermVectorsConsumer(docWriter); } termsHash = new FreqProxTermsWriter(docWriter, termVectorsWriter); }
info.info.getDiagnostics(), info.info.getId(), info.info.getAttributes(), info.info.getIndexSort()); SegmentCommitInfo newInfoPerCommit = new SegmentCommitInfo(newInfo, info.getDelCount(), info.getSoftDelCount(), info.getDelGen(), info.getFieldInfosGen(), info.getDocValuesGen());