/** Expert: low-level implementation method * Returns an Explanation that describes how <code>doc</code> scored against * <code>weight</code>. * * <p>This is intended to be used in developing Similarity implementations, * and, for good performance, should not be displayed with every hit. * Computing an explanation is as expensive as executing the query over the * entire index. * <p>Applications should call {@link IndexSearcher#explain(Query, int)}. * @throws BooleanQuery.TooManyClauses If a query would exceed * {@link BooleanQuery#getMaxClauseCount()} clauses. */ protected Explanation explain(Weight weight, int doc) throws IOException { int n = ReaderUtil.subIndex(doc, leafContexts); final LeafReaderContext ctx = leafContexts.get(n); int deBasedDoc = doc - ctx.docBase; final Bits liveDocs = ctx.reader().getLiveDocs(); if (liveDocs != null && liveDocs.get(deBasedDoc) == false) { return Explanation.noMatch("Document " + doc + " is deleted"); } return weight.explain(ctx, deBasedDoc); }
private boolean assertCheckLiveDocs(Bits bits, int expectedLength, int expectedDeleteCount) { assert bits.length() == expectedLength; int deletedCount = 0; for (int i = 0; i < bits.length(); i++) { if (bits.get(i) == false) { deletedCount++; } } assert deletedCount == expectedDeleteCount : "deleted: " + deletedCount + " != expected: " + expectedDeleteCount; return true; }
boolean hasDeletions = reader.hasDeletions(); Bits liveDocs = MultiFields.getLiveDocs(reader); if (reader.hasDeletions() && !liveDocs.get(i)) continue; // if it is deleted, just ignore it. Document d = reader.document(results.scoreDocs[i].doc); double distance = getDistance(d, globalFeature); assert (distance >= 0);
/** * Check whether there is one or more documents matching the provided query. */ public static boolean exists(IndexSearcher searcher, Query query) throws IOException { final Weight weight = searcher.createNormalizedWeight(query, false); // the scorer API should be more efficient at stopping after the first // match than the bulk scorer API for (LeafReaderContext context : searcher.getIndexReader().leaves()) { final Scorer scorer = weight.scorer(context); if (scorer == null) { continue; } final Bits liveDocs = context.reader().getLiveDocs(); final DocIdSetIterator iterator = scorer.iterator(); for (int doc = iterator.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iterator.nextDoc()) { if (liveDocs == null || liveDocs.get(doc)) { return true; } } } return false; }
int remainingBits = (int) Math.min(iterator.cost(), Integer.MAX_VALUE); DocIdSetBuilder.BulkAdder adder = builder == null ? null : builder.grow(remainingBits); final Bits liveDocs = context.reader().getLiveDocs(); final LeafBucketCollector collector = queue.getLeafCollector(leadSourceBucket, context, queueCollector); while (iterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { if (liveDocs == null || liveDocs.get(iterator.docID())) { collector.collect(iterator.docID());
static int countSoftDeletes(DocIdSetIterator softDeletedDocs, Bits hardDeletes) throws IOException { int count = 0; if (softDeletedDocs != null) { int doc; while ((doc = softDeletedDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { if (hardDeletes == null || hardDeletes.get(doc)) { count++; } } } return count; } }
protected Set<String> getShardDocUIDs(final IndexShard shard) throws IOException { shard.refresh("get_uids"); try (Engine.Searcher searcher = shard.acquireSearcher("test")) { Set<String> ids = new HashSet<>(); for (LeafReaderContext leafContext : searcher.reader().leaves()) { LeafReader reader = leafContext.reader(); Bits liveDocs = reader.getLiveDocs(); for (int i = 0; i < reader.maxDoc(); i++) { if (liveDocs == null || liveDocs.get(i)) { Document uuid = reader.document(i, Collections.singleton(IdFieldMapper.NAME)); BytesRef binaryID = uuid.getBinaryValue(IdFieldMapper.NAME); ids.add(Uid.decodeId(Arrays.copyOfRange(binaryID.bytes, binaryID.offset, binaryID.offset + binaryID.length))); } } } return ids; } }
private static boolean isDeleted(IndexReader reader, int docId) { if (reader.hasDeletions()) { List<LeafReaderContext> leaves = reader.leaves(); int size = leaves.size(); for (int i = 0; i < size; i++) { Bits liveDocs = leaves.get(i).reader().getLiveDocs(); if (docId < liveDocs.length()) { boolean isDeleted = !liveDocs.get(docId); if (isDeleted) { return true; } } } return false; } else { return false; } }
for (int i = 0; i < hits.length; ++i) { SearchHit hit = hits[i]; int hitReaderIndex = ReaderUtil.subIndex(hit.docId(), indexReader.leaves()); if (readerIndex != hitReaderIndex) { readerIndex = hitReaderIndex; LeafReaderContext ctx = indexReader.leaves().get(readerIndex); docBase = ctx.docBase; matchingDocs = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), scorerSupplier); if (matchingDocs.get(hit.docId() - docBase)) { matchedQueries[i].add(name);
@Override protected boolean match( int doc ) { return liveDocs.get( doc ); } };
if (filterWeights[i] != null) { final Bits docSet = Lucene.asSequentialAccessBits( context.reader().maxDoc(), filterWeights[i].scorerSupplier(context)); if (docSet.get(doc) == false) { continue; } else { FunctionFactorScorer scorer = functionScorer(context); int actualDoc = scorer.iterator().advance(doc); assert (actualDoc == doc); double score = scorer.computeScore(doc, expl.getValue());
assert context.reader().getCoreCacheHelper().getKey().equals(readerKey) : "context's reader is not the same as the reader class was initialized on."; final Bits liveDocs = context.reader().getLiveDocs(); DocIdAndSeqNo result = null; int docID = docsEnum.nextDoc(); if (docID != DocIdSetIterator.NO_MORE_DOCS) { final NumericDocValues seqNoDV = context.reader().getNumericDocValues(SeqNoFieldMapper.NAME); for (; docID != DocIdSetIterator.NO_MORE_DOCS; docID = docsEnum.nextDoc()) { final long seqNo; seqNo = UNASSIGNED_SEQ_NO; final boolean isLive = (liveDocs == null || liveDocs.get(docID)); if (isLive) {
@Override public int numDeletesToMerge(SegmentCommitInfo info, int delCount, IOSupplier<CodecReader> readerSupplier) throws IOException { final int numDeletesToMerge = super.numDeletesToMerge(info, delCount, readerSupplier); if (numDeletesToMerge != 0 && info.getSoftDelCount() > 0) { final CodecReader reader = readerSupplier.get(); if (reader.getLiveDocs() != null) { BooleanQuery.Builder builder = new BooleanQuery.Builder(); builder.add(new DocValuesFieldExistsQuery(field), BooleanClause.Occur.FILTER); builder.add(retentionQuerySupplier.get(), BooleanClause.Occur.FILTER); Scorer scorer = getScorer(builder.build(), FilterCodecReader.wrapLiveDocs(reader, null, reader.maxDoc())); if (scorer != null) { DocIdSetIterator iterator = scorer.iterator(); Bits liveDocs = reader.getLiveDocs(); int numDeletedDocs = reader.numDeletedDocs(); while (iterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { if (liveDocs.get(iterator.docID()) == false) { numDeletedDocs--; } } return numDeletedDocs; } } } assert numDeletesToMerge >= 0 : "numDeletesToMerge: " + numDeletesToMerge; assert numDeletesToMerge <= info.info.maxDoc() : "numDeletesToMerge: " + numDeletesToMerge + " maxDoc:" + info.info.maxDoc(); return numDeletesToMerge; } }
private PackedLongValues getDeletes(List<CodecReader> readers) { PackedLongValues.Builder deletes = PackedLongValues.monotonicBuilder(PackedInts.COMPACT); int deleteCount = 0; for (LeafReader reader : readers) { final int maxDoc = reader.maxDoc(); final Bits liveDocs = reader.getLiveDocs(); for (int i = 0; i < maxDoc; ++i) { if (liveDocs != null && !liveDocs.get(i)) { ++deleteCount; } else { deletes.add(deleteCount); } } } return deletes.build(); }
private static void addDocuments(LeafReader reader, Term term, Collection<Document> documents) throws IOException { PostingsEnum docs = reader.postings(term); if (docs != null) { int docId; while ((docId = docs.nextDoc()) != PostingsEnum.NO_MORE_DOCS) { Bits liveDocs = reader.getLiveDocs(); // Maybe some of the docs have been deleted! Check that too.. if (liveDocs != null && !liveDocs.get(docId)) { continue; } Document document = readDocument(reader, docId, null); documents.add(document); } } }
private static DocIdSet liveDocsBitSet(LeafReader reader, Term term) throws IOException { FixedBitSet bitSet = new FixedBitSet(reader.maxDoc()); Bits liveDocs = reader.getLiveDocs(); PostingsEnum termDocs = reader.postings(term); long setSize = 0; while (termDocs.nextDoc() != NO_MORE_DOCS) { int docID = termDocs.docID(); if (liveDocs == null || liveDocs.get(docID)) { bitSet.set(docID); ++setSize; // else document is deleted... } } return new BitDocIdSet(bitSet, setSize); } }
@Override public LeafReader wrap(LeafReader leaf) { SegmentReader segmentReader = segmentReader(leaf); Bits hardLiveDocs = segmentReader.getHardLiveDocs(); if (hardLiveDocs == null) { return new LeafReaderWithLiveDocs(leaf, null, leaf.maxDoc()); } // TODO: Can we avoid calculate numDocs by using SegmentReader#getSegmentInfo with LUCENE-8458? int numDocs = 0; for (int i = 0; i < hardLiveDocs.length(); i++) { if (hardLiveDocs.get(i)) { numDocs++; } } return new LeafReaderWithLiveDocs(segmentReader, hardLiveDocs, numDocs); } });
@Override public int length() { return in.length(); } }