IndexWriter indexWriter = new IndexWriter(index, new IndexWriterConfig(analyzer))) { final Document doc = new Document(); final Field v = new TextField(Fields.VENDOR, Fields.VENDOR, Field.Store.YES); final Field p = new TextField(Fields.PRODUCT, Fields.PRODUCT, Field.Store.YES); doc.add(v); doc.add(p); v.setStringValue(pair.getLeft()); p.setStringValue(pair.getRight()); indexWriter.addDocument(doc); productFieldAnalyzer.reset(); vendorFieldAnalyzer.reset(); indexWriter.commit(); } catch (DatabaseException ex) { LOGGER.debug("", ex);
IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_47, analyzer); iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); IndexWriter writer = new IndexWriter(dir, iwc); Document doc = new Document(); doc.add(field); if (writer.getConfig().getOpenMode() == OpenMode.CREATE) { writer.addDocument(doc); } else { writer.updateDocument(new Term("content",strs[i]), doc);
try { Analyzer analyzer = AnalyzerGuru.getAnalyzer(); IndexWriterConfig iwc = new IndexWriterConfig(analyzer); iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); iwc.setRAMBufferSizeMB(env.getRamBufferSize()); iwc.setCodec(new Lucene70Codec( Lucene50StoredFieldsFormat.Mode.BEST_COMPRESSION)); writer = new IndexWriter(indexDirectory, iwc); writer.commit(); // to make sure index exists on the disk completer = new PendingFileCompleter(); reader = DirectoryReader.open(indexDirectory); // open existing index settings = readAnalysisSettings(); if (settings == null) { try { if (writer != null) { writer.close();
/** Creates index upgrader on the given directory, using an {@link IndexWriter} using the given * {@code matchVersion}. You have the possibility to upgrade indexes with multiple commit points by removing * all older ones. If {@code infoStream} is not {@code null}, all logging output will be sent to this stream. */ public IndexUpgrader(Directory dir, InfoStream infoStream, boolean deletePriorCommits) { this(dir, new IndexWriterConfig(null), deletePriorCommits); if (null != infoStream) { this.iwc.setInfoStream(infoStream); } }
public static IndexWriterConfig population( Analyzer analyzer ) { IndexWriterConfig writerConfig = standard( analyzer ); writerConfig.setMaxBufferedDocs( POPULATION_MAX_BUFFERED_DOCS ); writerConfig.setRAMBufferSizeMB( POPULATION_RAM_BUFFER_SIZE_MB ); return writerConfig; }
private IndexWriter instantiateWriter( File folder ) { Directory dir = null; try { dir = LuceneDataSource.getDirectory( folder, identifier ); IndexWriterConfig writerConfig = new IndexWriterConfig( type.analyzer ); writerConfig.setRAMBufferSizeMB( determineGoodBufferSize( writerConfig.getRAMBufferSizeMB() ) ); return new IndexWriter( dir, writerConfig ); } catch ( IOException e ) { IOUtils.closeAllSilently( dir ); throw new RuntimeException( e ); } }
@Test public void testTextFVIndexAndSearch() throws Exception { String fieldName = "text"; String[] texts = new String[]{ "0.1,0.3,0.5,0.7,0.11,0.13,0.17,0.19,0.23,0.29", "0.1 0.3 0.5 0.7 0.11 0.13 0.17 0.19 0.23 0.29" }; for (String text : texts) { LSHAnalyzer analyzer = new LSHAnalyzer(); Directory directory = new RAMDirectory(); IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_47, analyzer)); DirectoryReader reader = null; try { Document document = new Document(); document.add(new TextField(fieldName, text, Field.Store.YES)); writer.addDocument(document); writer.commit(); reader = DirectoryReader.open(writer, false); assertSimQuery(analyzer, fieldName, text, reader); } finally { if (reader != null) { reader.close(); } writer.close(); directory.close(); } } }
public static void initializeDirectory(Directory directory) throws IOException { IndexWriterConfig indexWriterConfig = new IndexWriterConfig(LuceneSettings.analyzer); IndexWriter iwriter = new IndexWriter(directory, indexWriterConfig); iwriter.commit(); iwriter.close(); //reopen to check for index IndexReader reader = DirectoryReader.open(directory); reader.close(); }
/** * This method removes all lucene files from the given directory. It will first try to delete all commit points / segments * files to ensure broken commits or corrupted indices will not be opened in the future. If any of the segment files can't be deleted * this operation fails. */ public static void cleanLuceneIndex(Directory directory) throws IOException { try (Lock writeLock = directory.obtainLock(IndexWriter.WRITE_LOCK_NAME)) { for (final String file : directory.listAll()) { if (file.startsWith(IndexFileNames.SEGMENTS) || file.equals(IndexFileNames.OLD_SEGMENTS_GEN)) { directory.deleteFile(file); // remove all segment_N files } } } try (IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Lucene.STANDARD_ANALYZER) .setSoftDeletesField(Lucene.SOFT_DELETES_FIELD) .setMergePolicy(NoMergePolicy.INSTANCE) // no merges .setCommitOnClose(false) // no commits .setOpenMode(IndexWriterConfig.OpenMode.CREATE))) // force creation - don't append... { // do nothing and close this will kick of IndexFileDeleter which will remove all pending files } }
IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(indexPath))); int numDocs = reader.numDocs(); boolean hasDeletions = reader.hasDeletions(); int countUpdated = 0; IndexReader readerRo = DirectoryReader.open(FSDirectory.open(new File(indexPath + "-ro"))); ImageSearcher searcher = new GenericImageSearcher(numReferenceObjectsUsed, featureClass, featureFieldName); Map<String, Analyzer> perField = new HashMap<String, Analyzer>(1); new PerFieldAnalyzerWrapper(new SimpleAnalyzer(LuceneUtils.LUCENE_VERSION), perField); IndexWriter iw = new IndexWriter(FSDirectory.open(new File(indexPath)), new IndexWriterConfig(LuceneUtils.LUCENE_VERSION, aWrapper).setOpenMode(IndexWriterConfig.OpenMode.CREATE)); StringBuilder sb = new StringBuilder(256); document.add(new TextField("ro-order", sb.toString(), Field.Store.YES)); iw.updateDocument(new Term(DocumentBuilder.FIELD_NAME_IDENTIFIER, document.getValues(DocumentBuilder.FIELD_NAME_IDENTIFIER)[0]), document); countUpdated++; iw.commit(); iw.close();
public static IndexWriter createIndexWriter(Directory directory, boolean create, AnalyzerType analyzer, double RAMBufferSize) throws IOException { // set the analyzer according to the method params Analyzer tmpAnalyzer = null; if (analyzer == AnalyzerType.SimpleAnalyzer) tmpAnalyzer = new SimpleAnalyzer(); else if (analyzer == AnalyzerType.WhitespaceAnalyzer) tmpAnalyzer = new WhitespaceAnalyzer(); // The config IndexWriterConfig config = new IndexWriterConfig(tmpAnalyzer); if (create) config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); // overwrite if it exists. else config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); // create new if none is there, append otherwise. config.setRAMBufferSizeMB(RAMBufferSize); config.setCodec(new LireCustomCodec()); return new IndexWriter(directory, config); }
IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); int maxLen = 1000000; int len = 0; try (IndexWriter writer = new IndexWriter(directory, indexWriterConfig)) { List<Document> docs = new ArrayList<>(); try (BufferedReader reader = getReader(inputFile)) { writer.addDocuments(docs); writer.commit(); writer.flush(); try (IndexReader reader = DirectoryReader.open(directory)) { LeafReader wrappedReader = SlowCompositeReaderWrapper.wrap(reader); Terms terms = wrappedReader.terms(FIELD); TermsEnum termsEnum = terms.iterator(); BytesRef bytesRef = termsEnum.next(); int docsWThisField = wrappedReader.getDocCount(FIELD); while (bytesRef != null) { int df = termsEnum.docFreq();
/** * Initialize the Lucene Directory if it isn't already. * * @param directory the Directory to initialize * @throws SearchException in case of lock acquisition timeouts, IOException, or if a corrupt index is found */ public static void initializeIndexIfNeeded(Directory directory) { SimpleAnalyzer analyzer = new SimpleAnalyzer(); try { if ( ! DirectoryReader.indexExists( directory ) ) { try { IndexWriterConfig iwriterConfig = new IndexWriterConfig( analyzer ).setOpenMode( OpenMode.CREATE_OR_APPEND ); //Needs to have a timeout higher than zero to prevent race conditions over (network) RPCs //for distributed indexes (Infinispan but probably also NFS and similar) SleepingLockWrapper delayedDirectory = new SleepingLockWrapper( directory, 2000, 20 ); IndexWriter iw = new IndexWriter( delayedDirectory, iwriterConfig ); iw.close(); } catch (LockObtainFailedException lofe) { log.lockingFailureDuringInitialization( directory.toString() ); } } } catch (IOException e) { throw new SearchException( "Could not initialize index", e ); } finally { analyzer.close(); } }
@Override public void open() throws IOException { if (_closed) { IndexWriterConfig idxWriterConfig = new IndexWriterConfig(Version.LUCENE_43, new StandardAnalyzer(Version.LUCENE_43)); idxWriterConfig.setMergePolicy(new ZoieMergePolicy()); idxWriterConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); _idxWriter = new IndexWriter(_dir, idxWriterConfig); updateReader(); _closed = false; } }
@Override public IndexWriter openIndexWriter(Analyzer analyzer, Similarity similarity) throws IOException { if (_indexWriter != null) { return _indexWriter; } ZoieMergePolicy mergePolicy = new ZoieMergePolicy(); mergePolicy.setMergePolicyParams(_mergePolicyParams); mergePolicy.setUseCompoundFile(false); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, analyzer); config.setOpenMode(OpenMode.CREATE_OR_APPEND); config.setMergeScheduler(_mergeScheduler); config.setMergePolicy(mergePolicy); config.setReaderPooling(false); if (similarity != null) { config.setSimilarity(similarity); } config.setRAMBufferSizeMB(3); IndexWriter idxWriter = new IndexWriter(_directory, config); _indexWriter = idxWriter; return idxWriter; }
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_47, analyzer).setOpenMode(OpenMode.CREATE); final IndexWriter writer = new IndexWriter(directory, config); for (Stop stop : graphIndex.stopForId.values()) { addStop(writer, stop); addCorner(writer, sv); writer.close(); long elapsedTime = System.currentTimeMillis() - startTime; LOG.info("Built Lucene index in {} msec", elapsedTime); searcher = new IndexSearcher(DirectoryReader.open(directory)); } catch (Exception ex) { throw new RuntimeException("Lucene indexing failed.", ex);
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_40, analyzer); IndexWriter w = new IndexWriter(index, config); addDoc(w, "Lucene in Action", "193398817"); addDoc(w, "Lucene for Dummies", "55320055Z"); addDoc(w, "Managing Gigabytes", "55063554A"); addDoc(w, "The Art of Computer Science", "9900333X"); w.close(); IndexReader reader = DirectoryReader.open(index); IndexSearcher searcher = new IndexSearcher(reader); TopScoreDocCollector collector = TopScoreDocCollector.create(hitsPerPage, true); int docId = hits[i].doc; Document d = searcher.doc(docId); System.out.println((i + 1) + ". " + d.get("isbn") + "\t" + d.get("title"));
private void ensureLuceneDataInstantiated() { if ( this.directory == null ) { try { this.directory = new RAMDirectory(); IndexWriterConfig writerConfig = new IndexWriterConfig( index.type.analyzer ); this.writer = new IndexWriter( directory, writerConfig ); } catch ( IOException e ) { throw new RuntimeException( e ); } } }
Query q = qp.parse(luceneQuery); w.deleteDocuments(q); w.commit(); w.close(); IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); LogDocMergePolicy logDocMergePolicy = new LogDocMergePolicy(); logDocMergePolicy.setMergeFactor(1000); indexWriterConfig.setMergePolicy(logDocMergePolicy); w = new IndexWriter(index, indexWriterConfig);
LOGGER.log(Level.INFO, "Optimizing the index{0}", projectDetail); Analyzer analyzer = new StandardAnalyzer(); IndexWriterConfig conf = new IndexWriterConfig(analyzer); conf.setOpenMode(OpenMode.CREATE_OR_APPEND); wrt = new IndexWriter(indexDirectory, conf); wrt.forceMerge(1); // this is deprecated and not needed anymore elapsed.report(LOGGER, String.format("Done optimizing index%s", projectDetail)); synchronized (lock) { if (wrt != null) { try { wrt.close(); } catch (IOException e) { if (writerException == null) {