result.put("update_seq", getUpdateSequence(reader.getIndexCommit().getUserData())); final JSONArray fields = new JSONArray(); for (LeafReaderContext leaf : reader.leaves()) { for (FieldInfo info : leaf.reader().getFieldInfos()) { if (info.name.startsWith("_")) {
final Query query = LongPoint.newRangeQuery(SeqNoFieldMapper.NAME, fromSeqNo, toSeqNo); final Weight weight = searcher.createWeight(query, false, 1.0f); for (LeafReaderContext leaf : reader.leaves()) { final Scorer scorer = weight.scorer(leaf); if (scorer == null) {
@Override public IndexParentChildFieldData loadGlobal(DirectoryReader indexReader) { if (indexReader.leaves().size() <= 1) { // ordinals are already global return this; } try { return cache.load(indexReader, this); } catch (Throwable e) { if (e instanceof ElasticsearchException) { throw (ElasticsearchException) e; } else { throw new ElasticsearchException(e.getMessage(), e); } } }
public long globalMaxOrd(IndexSearcher indexSearcher, String type) { DirectoryReader indexReader = (DirectoryReader) indexSearcher.getIndexReader(); if (indexReader.leaves().isEmpty()) { return 0; } else { LeafReaderContext atomicReaderContext = indexReader.leaves().get(0); IndexParentChildFieldData globalFieldData = indexFieldData.loadGlobal(indexReader); AtomicParentChildFieldData afd = globalFieldData.load(atomicReaderContext); SortedDocValues values = afd.getOrdinalsValues(type); return values.getValueCount(); } }
static IndexSearcher createMultiDocumentSearcher(Analyzer analyzer, Collection<ParsedDocument> docs) { RAMDirectory ramDirectory = new RAMDirectory(); try (IndexWriter indexWriter = new IndexWriter(ramDirectory, new IndexWriterConfig(analyzer))) { // Indexing in order here, so that the user provided order matches with the docid sequencing: Iterable<ParseContext.Document> iterable = () -> docs.stream() .map(ParsedDocument::docs) .flatMap(Collection::stream) .iterator(); indexWriter.addDocuments(iterable); DirectoryReader directoryReader = DirectoryReader.open(indexWriter); assert directoryReader.leaves().size() == 1 : "Expected single leaf, but got [" + directoryReader.leaves().size() + "]"; final IndexSearcher slowSearcher = new IndexSearcher(directoryReader) { @Override public Weight createNormalizedWeight(Query query, boolean needsScores) throws IOException { BooleanQuery.Builder bq = new BooleanQuery.Builder(); bq.add(query, BooleanClause.Occur.MUST); bq.add(Queries.newNestedFilter(), BooleanClause.Occur.MUST_NOT); return super.createNormalizedWeight(bq.build(), needsScores); } }; slowSearcher.setQueryCache(null); return slowSearcher; } catch (IOException e) { throw new ElasticsearchException("Failed to create index for percolator with nested document ", e); } }
/** * Creates a corpus adapter which uses the Lucene index with the given path * and searches on the field with the given field name. * * @param indexPath * @param fieldName * @return * @throws CorruptIndexException * @throws IOException */ public static LuceneCorpusAdapter create(String indexPath, String fieldName) throws CorruptIndexException, IOException { DirectoryReader dirReader = DirectoryReader.open(new NIOFSDirectory(new File(indexPath))); List<AtomicReaderContext> leaves = dirReader.leaves(); AtomicReader reader[] = new AtomicReader[leaves.size()]; AtomicReaderContext contexts[] = new AtomicReaderContext[leaves.size()]; for (int i = 0; i < reader.length; i++) { contexts[i] = leaves.get(i); reader[i] = contexts[i].reader(); } return new LuceneCorpusAdapter(dirReader, reader, contexts, fieldName); }
private LeafReader build(IndexWriter writer) throws IOException { for (InputDocument doc : documents) { writer.addDocument(doc.getDocument()); } writer.commit(); writer.forceMerge(1); LeafReader reader = DirectoryReader.open(directory).leaves().get(0).reader(); assert reader != null; docIds = new String[reader.maxDoc()]; for (int i = 0; i < docIds.length; i++) { docIds[i] = reader.document(i).get(InputDocument.ID_FIELD); // TODO can this be more efficient? } return reader; }
public GetFieldNamesResponse getFieldNames() throws IOException { openReaderIfChanges(); GetFieldNamesResponse.Builder builder = GetFieldNamesResponse.newBuilder(); Set<String> fields = new HashSet<>(); for (LeafReaderContext subReaderContext : directoryReader.leaves()) { FieldInfos fieldInfos = subReaderContext.reader().getFieldInfos(); for (FieldInfo fi : fieldInfos) { String fieldName = fi.name; fields.add(fieldName); } } fields.forEach(builder::addFieldName); return builder.build(); }
private void createIndex(IndexWriterConfig config, Directory target, DirectoryReader reader, Query preserveFilter, boolean negateFilter) throws IOException { boolean success = false; final IndexWriter w = new IndexWriter(target, config); try { final IndexSearcher searcher = new IndexSearcher(reader); searcher.setQueryCache(null); final boolean needsScores = false; // scores are not needed, only matching docs final Weight preserveWeight = searcher.createNormalizedWeight(preserveFilter, needsScores); final List<LeafReaderContext> leaves = reader.leaves(); final CodecReader[] subReaders = new CodecReader[leaves.size()]; int i = 0; for (final LeafReaderContext ctx : leaves) { subReaders[i++] = new DocumentFilteredLeafIndexReader(ctx, preserveWeight, negateFilter); } w.addIndexes(subReaders); success = true; } finally { if (success) { w.close(); } else { IOUtils.closeWhileHandlingException(w); } } }
@Override public synchronized Collection<Accountable> getChildResources() { final List<Accountable> resources = new ArrayList<>(); long ramBytesUsed = 0; for (LeafReaderContext ctx : indexReader.leaves()) { ramBytesUsed += ((SegmentReader) ctx.reader()).ramBytesUsed(); } resources.add(Accountables.namedAccountable("indexReader", ramBytesUsed)); if (taxoArrays != null) { resources.add(Accountables.namedAccountable("taxoArrays", taxoArrays)); } synchronized (categoryCache) { resources.add(Accountables.namedAccountable("categoryCache", BYTES_PER_CACHE_ENTRY * categoryCache.size())); } synchronized (ordinalCache) { resources.add(Accountables.namedAccountable("ordinalCache", BYTES_PER_CACHE_ENTRY * ordinalCache.size())); } return Collections.unmodifiableList(resources); }
private void createIndex(IndexWriterConfig config, Directory target, DirectoryReader reader, Query preserveFilter, boolean negateFilter) throws IOException { boolean success = false; final IndexWriter w = new IndexWriter(target, config); try { final IndexSearcher searcher = new IndexSearcher(reader); searcher.setQueryCache(null); preserveFilter = searcher.rewrite(preserveFilter); final Weight preserveWeight = searcher.createWeight(preserveFilter, false, 1); final List<LeafReaderContext> leaves = reader.leaves(); final CodecReader[] subReaders = new CodecReader[leaves.size()]; int i = 0; for (final LeafReaderContext ctx : leaves) { subReaders[i++] = new DocumentFilteredLeafIndexReader(ctx, preserveWeight, negateFilter); } w.addIndexes(subReaders); success = true; } finally { if (success) { w.close(); } else { IOUtils.closeWhileHandlingException(w); } } }
/** * Merges the given taxonomy and index directories and commits the changes to * the given writers. */ public static void merge(Directory srcIndexDir, Directory srcTaxoDir, OrdinalMap map, IndexWriter destIndexWriter, DirectoryTaxonomyWriter destTaxoWriter, FacetsConfig srcConfig) throws IOException { // merge the taxonomies destTaxoWriter.addTaxonomy(srcTaxoDir, map); int ordinalMap[] = map.getMap(); DirectoryReader reader = DirectoryReader.open(srcIndexDir); try { List<LeafReaderContext> leaves = reader.leaves(); int numReaders = leaves.size(); CodecReader wrappedLeaves[] = new CodecReader[numReaders]; for (int i = 0; i < numReaders; i++) { wrappedLeaves[i] = SlowCodecReaderWrapper.wrap(new OrdinalMappingLeafReader(leaves.get(i).reader(), ordinalMap, srcConfig)); } destIndexWriter.addIndexes(wrappedLeaves); // commit changes to taxonomy and index respectively. destTaxoWriter.commit(); destIndexWriter.commit(); } finally { reader.close(); } }
/** * Merges the given taxonomy and index directories and commits the changes to * the given writers. */ public static void merge(Directory srcIndexDir, Directory srcTaxoDir, OrdinalMap map, IndexWriter destIndexWriter, DirectoryTaxonomyWriter destTaxoWriter, FacetsConfig srcConfig) throws IOException { // merge the taxonomies destTaxoWriter.addTaxonomy(srcTaxoDir, map); int ordinalMap[] = map.getMap(); DirectoryReader reader = DirectoryReader.open(srcIndexDir); try { List<LeafReaderContext> leaves = reader.leaves(); int numReaders = leaves.size(); CodecReader wrappedLeaves[] = new CodecReader[numReaders]; for (int i = 0; i < numReaders; i++) { wrappedLeaves[i] = SlowCodecReaderWrapper.wrap(new OrdinalMappingLeafReader(leaves.get(i).reader(), ordinalMap, srcConfig)); } destIndexWriter.addIndexes(wrappedLeaves); // commit changes to taxonomy and index respectively. destTaxoWriter.commit(); destIndexWriter.commit(); } finally { reader.close(); } }
@Override public synchronized long ramBytesUsed() { ensureOpen(); long ramBytesUsed = 0; for (LeafReaderContext ctx : indexReader.leaves()) { ramBytesUsed += ((SegmentReader) ctx.reader()).ramBytesUsed(); } if (taxoArrays != null) { ramBytesUsed += taxoArrays.ramBytesUsed(); } synchronized (categoryCache) { ramBytesUsed += BYTES_PER_CACHE_ENTRY * categoryCache.size(); } synchronized (ordinalCache) { ramBytesUsed += BYTES_PER_CACHE_ENTRY * ordinalCache.size(); } return ramBytesUsed; }
public int getNumberOfSegments() { return this.getCore().getSearcher().get().getIndexReader().leaves().size(); }
public int getNumberOfSegments() { return this.getCore().getSearcher().get().getIndexReader().leaves().size(); }
/** * Read all terms from a field * * @param field the field in the document to load terms from * @param directory Any directory implementation * @return Unique terms represented as UTF-8 * @throws IOException */ public static Set<String> readTerms(String field, Directory directory) throws IOException { try (DirectoryReader reader = DirectoryReader.open(directory)) { Set<String> termStrings = new TreeSet<>(); for (LeafReaderContext atomicReaderContext : reader.leaves()) { LeafReader atomicReader = atomicReaderContext.reader(); TermsEnum iterator = atomicReader.terms(field).iterator(); BytesRef next = iterator.next(); while (next != null) { termStrings.add(iterator.term().utf8ToString()); next = iterator.next(); } } return termStrings; } }