System.out.println("Indexing to directory '" + indexPath + "'..."); Date start = new Date(); Directory dir = FSDirectory.open(new File(indexPath));//Dirctory dir-->FSDirectory IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_47, analyzer); iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); IndexWriter writer = new IndexWriter(dir, iwc); Document doc = new Document(); doc.add(field); if (writer.getConfig().getOpenMode() == OpenMode.CREATE) { writer.addDocument(doc); } else { writer.updateDocument(new Term("content",strs[i]), doc); writer.close();
private void addPatterns(String id, Map<Integer, Set<E>> p, boolean commit) { try{ setIndexWriter(); Document doc = new Document(); doc.add(new StringField("sentid", id, Field.Store.YES)); doc.add(new Field("patterns", getBytes(p), LuceneFieldType.NOT_INDEXED)); indexWriter.addDocument(doc); if(commit){ indexWriter.commit(); //closeIndexWriter(); } }catch(IOException e){ throw new RuntimeException(e); } }
/** * Writes a document to contain the serialized version of {@code settings}, * with a {@link QueryBuilder#OBJUID} value set to * {@link #INDEX_ANALYSIS_SETTINGS_OBJUID}. An existing version of the * document is first deleted. * @param writer a defined, target instance * @param settings a defined instance * @throws IOException if I/O error occurs while writing Lucene */ public void write(IndexWriter writer, IndexAnalysisSettings settings) throws IOException { byte[] objser = settings.serialize(); writer.deleteDocuments(new Term(QueryBuilder.OBJUID, INDEX_ANALYSIS_SETTINGS_OBJUID)); Document doc = new Document(); StringField uidfield = new StringField(QueryBuilder.OBJUID, INDEX_ANALYSIS_SETTINGS_OBJUID, Field.Store.NO); doc.add(uidfield); doc.add(new StoredField(QueryBuilder.OBJSER, objser)); doc.add(new StoredField(QueryBuilder.OBJVER, INDEX_ANALYSIS_SETTINGS_OBJVER)); writer.addDocument(doc); }
private void applyDocuments( IndexWriter writer, IndexType type, LongObjectMap<DocumentContext> documents ) throws IOException { for ( DocumentContext context : documents ) { if ( context.exists ) { if ( LuceneDataSource.documentIsEmpty( context.document ) ) { writer.deleteDocuments( type.idTerm( context.entityId ) ); } else { writer.updateDocument( type.idTerm( context.entityId ), context.document ); } } else { writer.addDocument( context.document ); } } }
private static void writeSomething( IndexReference indexReference ) throws IOException { IndexWriter writer = indexReference.getWriter(); writer.addDocument( new Document() ); writer.commit(); }
/** Build the example index. */ public void index() throws IOException { IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig( new WhitespaceAnalyzer()).setOpenMode(OpenMode.CREATE)); // Add documents with a fake timestamp, 1000 sec before // "now", 2000 sec before "now", ...: for(int i=0;i<100;i++) { Document doc = new Document(); long then = nowSec - i * 1000; // Add as doc values field, so we can compute range facets: doc.add(new NumericDocValuesField("timestamp", then)); // Add as numeric field so we can drill-down: doc.add(new LongPoint("timestamp", then)); indexWriter.addDocument(doc); } // Open near-real-time searcher searcher = new IndexSearcher(DirectoryReader.open(indexWriter)); indexWriter.close(); }
private void indexTestDocuments(Directory directory) throws IOException { IndexWriterConfig indexWriterConfig = new IndexWriterConfig( new StandardAnalyzer() ); indexWriterConfig.setOpenMode( IndexWriterConfig.OpenMode.CREATE ); IndexWriter indexWriter = new IndexWriter( directory, indexWriterConfig ); Document document = new Document(); document.add( new StringField( "stringField", "test", Field.Store.NO ) ); document.add( new IntField( "intField", 0, Field.Store.NO ) ); indexWriter.addDocument( document ); indexWriter.commit(); indexWriter.close(); }
private void initLucene() throws IOException { StandardAnalyzer analyzer = new StandardAnalyzer(Version.LUCENE_43); Directory index = new RAMDirectory(); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, analyzer); IndexWriter writer = new IndexWriter(index, config); for (String s : classList.keySet()) { Document doc = new Document(); doc.add(new org.apache.lucene.document.TextField("classname", s, Field.Store.YES)); writer.addDocument(doc); } writer.close(); queryParser = new QueryParser(Version.LUCENE_43, "classname", analyzer); searcher = new IndexSearcher(DirectoryReader.open(index)); }
private static Directory createSampleDirectory(long numOfDocs, Iterable<Document> docs) throws IOException { Directory dir = new RAMDirectory(); IndexWriterConfig config = new IndexWriterConfig(VERSION, LuceneIndexConstants.ANALYZER); IndexWriter writer = new IndexWriter(dir, config); for (int i = 0; i < numOfDocs; i++) { Document doc = new Document(); doc.add(new StringField("foo", "bar" + i, Field.Store.NO)); writer.addDocument(doc); } for (Document doc : docs) { writer.addDocument(doc); } writer.close(); return dir; }
/** * Get the index writer/searcher wrapper for the given connection. * * @param conn the connection * @return the index access wrapper */ protected static IndexAccess getIndexAccess(Connection conn) throws SQLException { String path = getIndexPath(conn); synchronized (INDEX_ACCESS) { IndexAccess access = INDEX_ACCESS.get(path); if (access == null) { try { Directory indexDir = path.startsWith(IN_MEMORY_PREFIX) ? new RAMDirectory() : FSDirectory.open(new File(path)); Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_30); IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_30, analyzer); conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); IndexWriter writer = new IndexWriter(indexDir, conf); //see http://wiki.apache.org/lucene-java/NearRealtimeSearch access = new IndexAccess(writer); } catch (IOException e) { throw convertException(e); } INDEX_ACCESS.put(path, access); } return access; } }
IndexWriter indexWriter = new IndexWriter(index, new IndexWriterConfig(analyzer))) { final Document doc = new Document(); final Field v = new TextField(Fields.VENDOR, Fields.VENDOR, Field.Store.YES); final Field p = new TextField(Fields.PRODUCT, Fields.PRODUCT, Field.Store.YES); doc.add(v); doc.add(p); v.setStringValue(pair.getLeft()); p.setStringValue(pair.getRight()); indexWriter.addDocument(doc); productFieldAnalyzer.reset(); vendorFieldAnalyzer.reset(); indexWriter.commit(); } catch (DatabaseException ex) { LOGGER.debug("", ex);
try { Analyzer analyzer = AnalyzerGuru.getAnalyzer(); IndexWriterConfig iwc = new IndexWriterConfig(analyzer); iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); iwc.setRAMBufferSizeMB(env.getRamBufferSize()); iwc.setCodec(new Lucene70Codec( Lucene50StoredFieldsFormat.Mode.BEST_COMPRESSION)); writer = new IndexWriter(indexDirectory, iwc); writer.commit(); // to make sure index exists on the disk completer = new PendingFileCompleter(); reader = DirectoryReader.open(indexDirectory); // open existing index settings = readAnalysisSettings(); if (settings == null) { int numDocs = reader.numDocs(); if (numDocs > 0) { reader.close(); try { if (writer != null) { writer.close();
IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); int maxLen = 1000000; int len = 0; try (IndexWriter writer = new IndexWriter(directory, indexWriterConfig)) { List<Document> docs = new ArrayList<>(); try (BufferedReader reader = getReader(inputFile)) { while (line != null) { len += line.length(); Document document = new Document(); document.add(new TextField(FIELD, line, Field.Store.NO)); docs.add(document); if (len > maxLen) { writer.addDocuments(docs); docs.clear(); len = 0; writer.addDocuments(docs); writer.commit(); writer.flush(); try (IndexReader reader = DirectoryReader.open(directory)) { LeafReader wrappedReader = SlowCompositeReaderWrapper.wrap(reader); Terms terms = wrappedReader.terms(FIELD);
long startTime = System.currentTimeMillis(); directory = FSDirectory.open(new File(basePath, "lucene")); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_47, analyzer).setOpenMode(OpenMode.CREATE); final IndexWriter writer = new IndexWriter(directory, config); for (Stop stop : graphIndex.stopForId.values()) { addStop(writer, stop); addCorner(writer, sv); writer.close(); long elapsedTime = System.currentTimeMillis() - startTime; LOG.info("Built Lucene index in {} msec", elapsedTime); searcher = new IndexSearcher(DirectoryReader.open(directory)); } catch (Exception ex) { throw new RuntimeException("Lucene indexing failed.", ex);
Query query = new TermQuery(new Term(GRAMMED_WORDS_FIELD, term)); Sort sort = new Sort(COUNT_FIELD, true); List<String> suggestions = new ArrayList<String>(); for (ScoreDoc doc : docs.scoreDocs) { suggestions.add(autoCompleteReader.document(doc.doc).get( SOURCE_WORD_FIELD)); writer.setMergeFactor(300); writer.setMaxBufferedDocs(150); wordsMap.put(word, sourceReader.docFreq(new Term( fieldToAutocomplete, word))); writer.addDocument(doc); sourceReader.close(); writer.optimize(); writer.close();
Object val = v.isPlatformType() ? v.value(coctx, false) : v; Document doc = new Document(); doc.add(new TextField(VAL_STR_FIELD_NAME, val.toString(), Field.Store.YES)); doc.add(new TextField(idxdFields[i], fieldVal.toString(), Field.Store.YES)); final Term term = new Term(KEY_FIELD_NAME, keyByteRef); writer.deleteDocuments(term); doc.add(new StringField(KEY_FIELD_NAME, keyByteRef, Field.Store.YES)); writer.updateDocument(term, doc);
String projectDetail = this.project != null ? " for project " + project.getName() : ""; LOGGER.log(Level.INFO, "Optimizing the index{0}", projectDetail); Analyzer analyzer = new StandardAnalyzer(); IndexWriterConfig conf = new IndexWriterConfig(analyzer); conf.setOpenMode(OpenMode.CREATE_OR_APPEND); wrt = new IndexWriter(indexDirectory, conf); wrt.forceMerge(1); // this is deprecated and not needed anymore elapsed.report(LOGGER, String.format("Done optimizing index%s", projectDetail)); synchronized (lock) { if (wrt != null) { try { wrt.close(); } catch (IOException e) { if (writerException == null) {
QueryParser qp = new QueryParser(DEFAULT_SEARCHABLE_FIELD, new StandardAnalyzer()); Query q = qp.parse(luceneQuery); w.deleteDocuments(q); w.commit(); w.close(); IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); LogDocMergePolicy logDocMergePolicy = new LogDocMergePolicy(); logDocMergePolicy.setMergeFactor(1000); indexWriterConfig.setMergePolicy(logDocMergePolicy); w = new IndexWriter(index, indexWriterConfig); w.getConfig().setRAMBufferSizeMB(32);
public static void initializeDirectory(Directory directory) throws IOException { IndexWriterConfig indexWriterConfig = new IndexWriterConfig(LuceneSettings.analyzer); IndexWriter iwriter = new IndexWriter(directory, indexWriterConfig); iwriter.commit(); iwriter.close(); //reopen to check for index IndexReader reader = DirectoryReader.open(directory); reader.close(); }
tmpAnalyzer = new KeywordAnalyzer(); // entire string as one token. else if (analyzer == AnalyzerType.StandardAnalyzer) tmpAnalyzer = new StandardAnalyzer(); IndexWriterConfig config = new IndexWriterConfig(tmpAnalyzer); config.setRAMBufferSizeMB(512); config.setCommitOnClose(true); if (create) config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); // overwrite if it exists. config.setCodec(new LireCustomCodec()); return new IndexWriter(directory, config);