Analyzer analyzer = new FNLPAnalyzer(Version.LUCENE_47); DirectoryReader ireader = DirectoryReader.open(dir); IndexSearcher isearcher = new IndexSearcher(ireader); ScoreDoc[] hits = isearcher.search(query, null, 1000).scoreDocs; Document hitDoc = isearcher.doc(hits[i].doc); System.out.println(hitDoc.get("content")); System.out.println(hits[i].score); ireader.close(); dir.close();
/** * Get an indexReader for the Index database where a given file * * @param path the file to get the database for * @return The index database where the file should be located or null if it * cannot be located. */ public static IndexReader getIndexReader(String path) { IndexReader ret = null; RuntimeEnvironment env = RuntimeEnvironment.getInstance(); File indexDir = new File(env.getDataRootFile(), INDEX_DIR); if (env.hasProjects()) { Project p = Project.getProject(path); if (p == null) { return null; } indexDir = new File(indexDir, p.getPath()); } try { FSDirectory fdir = FSDirectory.open(indexDir.toPath(), NoLockFactory.INSTANCE); if (indexDir.exists() && DirectoryReader.indexExists(fdir)) { ret = DirectoryReader.open(fdir); } } catch (Exception ex) { LOGGER.log(Level.SEVERE, "Failed to open index: {0}", indexDir.getAbsolutePath()); LOGGER.log(Level.FINE, "Stack Trace: ", ex); } return ret; }
void setIndexReaderSearcher() throws IOException { FSDirectory index = FSDirectory.open(indexDir.toPath()); if(reader == null){ reader = DirectoryReader.open(index); searcher = new IndexSearcher(reader); }else{ DirectoryReader newreader = DirectoryReader.openIfChanged(reader); if(newreader != null) { reader.close(); reader = newreader; searcher = new IndexSearcher(reader); } } }
@Override public List<Document> call() { final List<Document> localScoreDocs = new ArrayList<>(); try (final DirectoryReader directoryReader = DirectoryReader.open(FSDirectory.open(indexDirectory))) { final IndexSearcher searcher = new IndexSearcher(directoryReader); final TopDocs topDocs = searcher.search(luceneQuery, 10000000); logger.info("For {}, Top Docs has {} hits; reading Lucene results", indexDirectory, topDocs.scoreDocs.length); if (topDocs.totalHits > 0) { for (final ScoreDoc scoreDoc : topDocs.scoreDocs) { final int docId = scoreDoc.doc; final Document d = directoryReader.document(docId); localScoreDocs.add(d); } } hits.addAndGet(localScoreDocs.size()); } catch (final IndexNotFoundException e) { } catch (final IOException ioe) { throw new RuntimeException(ioe); } return localScoreDocs; } };
DirectoryReader.open( this.writer ) : DirectoryReader.openIfChanged( (DirectoryReader) this.reader ); if ( newReader == null ) this.reader = newReader; LuceneUtil.close( searcher ); searcher = new IndexSearcher( reader );
IndexWriterConfig iwConfig = new IndexWriterConfig(analyzer); iwConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); iwriter = new IndexWriter(directory, iwConfig); Document doc = new Document(); doc.add(new StringField("ID", "10000", Field.Store.YES)); doc.add(new TextField(fieldName, text, Field.Store.YES)); iwriter.addDocument(doc); iwriter.close(); ireader = DirectoryReader.open(directory); isearcher = new IndexSearcher(ireader); TopDocs topDocs = isearcher.search(query, 5); System.out.println("命中:" + topDocs.totalHits); Document targetDoc = isearcher.doc(scoreDocs[i].doc); System.out.println("内容:" + targetDoc.toString()); if (ireader != null) { try { ireader.close(); } catch (IOException e) { e.printStackTrace(); directory.close(); } catch (IOException e) { e.printStackTrace();
private void initLucene() throws IOException { StandardAnalyzer analyzer = new StandardAnalyzer(Version.LUCENE_43); Directory index = new RAMDirectory(); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, analyzer); IndexWriter writer = new IndexWriter(index, config); for (String s : classList.keySet()) { Document doc = new Document(); doc.add(new org.apache.lucene.document.TextField("classname", s, Field.Store.YES)); writer.addDocument(doc); } writer.close(); queryParser = new QueryParser(Version.LUCENE_43, "classname", analyzer); searcher = new IndexSearcher(DirectoryReader.open(index)); }
IndexWriterConfig iwc = new IndexWriterConfig(LUCENE_41, anal); IndexWriter writer = new IndexWriter(idx, iwc); Document doc = new Document(); doc.add(new TextField(CONTENT_FIELD, haystack, Store.YES)); writer.addDocument(doc); writer.close(); IndexReader reader = open(idx); Query query = parser.parse(getNGram(needle, 3)); TopDocs results = new IndexSearcher(reader).search(query, 1); float score = 0; for (ScoreDoc hit : results.scoreDocs) { score += hit.score; reader.close(); idx.close(); return score;
@Test public void testTextFVIndexAndSearch() throws Exception { String fieldName = "text"; String[] texts = new String[]{ "0.1,0.3,0.5,0.7,0.11,0.13,0.17,0.19,0.23,0.29", "0.1 0.3 0.5 0.7 0.11 0.13 0.17 0.19 0.23 0.29" }; for (String text : texts) { LSHAnalyzer analyzer = new LSHAnalyzer(); Directory directory = new RAMDirectory(); IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_47, analyzer)); DirectoryReader reader = null; try { Document document = new Document(); document.add(new TextField(fieldName, text, Field.Store.YES)); writer.addDocument(document); writer.commit(); reader = DirectoryReader.open(writer, false); assertSimQuery(analyzer, fieldName, text, reader); } finally { if (reader != null) { reader.close(); } writer.close(); directory.close(); } } }
/** * Search one index. This is used if no projects are set up. * @param paging whether to use paging (if yes, first X pages will load * faster) * @param root which db to search * @throws IOException */ private void searchSingleDatabase(File root, boolean paging) throws IOException { IndexReader ireader = DirectoryReader.open(FSDirectory.open(root.toPath())); searcher = new IndexSearcher(ireader); collector = TopScoreDocCollector.create(hitsPerPage * cachePages); searcher.search(query, collector); totalHits = collector.getTotalHits(); if (!paging && totalHits > 0) { collector = TopScoreDocCollector.create(totalHits); searcher.search(query, collector); } hits = collector.topDocs().scoreDocs; for (ScoreDoc hit : hits) { int docId = hit.doc; Document d = searcher.doc(docId); docs.add(d); } }
IndexWriterConfig iwc = new IndexWriterConfig(indexAnalyzer); iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE); iwc.setRAMBufferSizeMB(ramBufferSizeMB); IndexWriter writer = new IndexWriter(dir, iwc); ft.freeze(); Document doc = new Document(); Field field = new Field("body", "", ft); doc.add(field); break; field.setStringValue(surfaceForm.utf8ToString()); writer.addDocument(doc); count++; reader = DirectoryReader.open(writer);
try { Analyzer analyzer = AnalyzerGuru.getAnalyzer(); IndexWriterConfig iwc = new IndexWriterConfig(analyzer); iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); iwc.setRAMBufferSizeMB(env.getRamBufferSize()); reader = DirectoryReader.open(indexDirectory); // open existing index settings = readAnalysisSettings(); if (settings == null) { int numDocs = reader.numDocs(); if (numDocs > 0) { if (terms != null) { uidIter = terms.iterator(); TermsEnum.SeekStatus stat = uidIter.seekCeil(new BytesRef(startuid)); //init uid if (stat == TermsEnum.SeekStatus.END) { uidIter = null; && uidIter.term().utf8ToString().startsWith(startuid)) { reader.close();
IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); int maxLen = 1000000; int len = 0; while (line != null) { len += line.length(); Document document = new Document(); document.add(new TextField(FIELD, line, Field.Store.NO)); docs.add(document); if (len > maxLen) { writer.flush(); try (IndexReader reader = DirectoryReader.open(directory)) { LeafReader wrappedReader = SlowCompositeReaderWrapper.wrap(reader); Terms terms = wrappedReader.terms(FIELD); TermsEnum termsEnum = terms.iterator(); BytesRef bytesRef = termsEnum.next(); int docsWThisField = wrappedReader.getDocCount(FIELD); while (bytesRef != null) { int df = termsEnum.docFreq(); String t = bytesRef.utf8ToString(); if (! WHITE_LIST.contains(t) && ! BLACK_LIST.contains(t)) { queue.insertWithOverflow(new TokenDFTF(t, df, tf));
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_47, analyzer).setOpenMode(OpenMode.CREATE); final IndexWriter writer = new IndexWriter(directory, config); for (Stop stop : graphIndex.stopForId.values()) { LOG.info("Built Lucene index in {} msec", elapsedTime); searcher = new IndexSearcher(DirectoryReader.open(directory)); } catch (Exception ex) { throw new RuntimeException("Lucene indexing failed.", ex);
private void testSearchSpeed(Class<? extends GlobalFeature> featureClass) throws IOException { ParallelIndexer parallelIndexer = new ParallelIndexer(DocumentBuilder.NUM_OF_THREADS, indexPath, testExtensive, true); parallelIndexer.addExtractor(featureClass); parallelIndexer.run(); IndexReader reader = DirectoryReader.open(new RAMDirectory(FSDirectory.open(Paths.get(indexPath)), IOContext.READONCE)); Bits liveDocs = MultiFields.getLiveDocs(reader); double queryCount = 0d; ImageSearcher searcher = new GenericFastImageSearcher(100, featureClass); long ms = System.currentTimeMillis(); String fileName; Document queryDoc; ImageSearchHits hits; for (int i = 0; i < reader.maxDoc(); i++) { if (reader.hasDeletions() && !liveDocs.get(i)) continue; // if it is deleted, just ignore it. fileName = getIDfromFileName(reader.document(i).getValues(DocumentBuilder.FIELD_NAME_IDENTIFIER)[0]); if (queries.keySet().contains(fileName)) { queryCount += 1d; // ok, we've got a query here for a document ... queryDoc = reader.document(i); hits = searcher.search(queryDoc, reader); } } ms = System.currentTimeMillis() - ms; System.out.printf("%s \t %3.1f \n", featureClass.getName().substring(featureClass.getName().lastIndexOf('.') + 1), (double) ms / queryCount); }
private LuceneSearcher(File indexDir) throws IOException { Path path = indexDir.toPath(); // symlinks are not supported here, see https://issues.apache.org/jira/browse/LUCENE-6700, // so we resolve the link ourselves: if (Files.isSymbolicLink(path)) { path = indexDir.getCanonicalFile().toPath(); } this.directory = FSDirectory.open(path); this.reader = DirectoryReader.open(directory); this.searcher = new IndexSearcher(reader); } public IndexReader getReader() {
public void listTokens(int freq) throws IOException { IndexReader ireader = null; TermsEnum iter = null; Terms terms; try { ireader = DirectoryReader.open(indexDirectory); int numDocs = ireader.numDocs(); if (numDocs > 0) { Fields uFields = MultiFields.getFields(ireader);//reader.getTermVectors(0); terms = uFields.terms(QueryBuilder.DEFS); iter = terms.iterator(); // init uid iterator } while (iter != null && iter.term() != null) { //if (iter.term().field().startsWith("f")) { if (iter.docFreq() > 16 && iter.term().utf8ToString().length() > freq) { LOGGER.warning(iter.term().utf8ToString()); } BytesRef next = iter.next(); if (next==null) {iter=null;} } } finally { if (ireader != null) { try { ireader.close(); } catch (IOException e) { LOGGER.log(Level.WARNING, "An error occurred while closing index reader", e); } } } }
final int refCount = searcher.getSearcher().getIndexSearcher().getIndexReader().getRefCount(); if (refCount <= 0) { final DirectoryReader directoryReader = DirectoryReader.open(directory); final IndexSearcher searcher = new IndexSearcher(directoryReader); final EventIndexSearcher eventIndexSearcher = new LuceneEventIndexSearcher(searcher, indexDir, directory, directoryReader); directory.close(); } catch (final IOException ioe) { e.addSuppressed(ioe); final DirectoryReader directoryReader = DirectoryReader.open(writer.getIndexWriter(), false); final IndexSearcher searcher = new IndexSearcher(directoryReader); final EventIndexSearcher eventIndexSearcher = new LuceneEventIndexSearcher(searcher, indexDir, null, directoryReader);
/** * Get number of documents in this index database. * @return number of documents * @throws IOException if I/O exception occurred */ public int getNumFiles() throws IOException { IndexReader ireader = null; int numDocs = 0; try { ireader = DirectoryReader.open(indexDirectory); // open existing index numDocs = ireader.numDocs(); } finally { if (ireader != null) { try { ireader.close(); } catch (IOException e) { LOGGER.log(Level.WARNING, "An error occurred while closing index reader", e); } } } return numDocs; }
public static void initializeDirectory(Directory directory) throws IOException { IndexWriterConfig indexWriterConfig = new IndexWriterConfig(LuceneSettings.analyzer); IndexWriter iwriter = new IndexWriter(directory, indexWriterConfig); iwriter.commit(); iwriter.close(); //reopen to check for index IndexReader reader = DirectoryReader.open(directory); reader.close(); }