public static IndexWriterConfig population( Analyzer analyzer ) { IndexWriterConfig writerConfig = standard( analyzer ); writerConfig.setMaxBufferedDocs( POPULATION_MAX_BUFFERED_DOCS ); writerConfig.setRAMBufferSizeMB( POPULATION_RAM_BUFFER_SIZE_MB ); return writerConfig; }
private IndexWriter instantiateWriter( File folder ) { Directory dir = null; try { dir = LuceneDataSource.getDirectory( folder, identifier ); IndexWriterConfig writerConfig = new IndexWriterConfig( type.analyzer ); writerConfig.setRAMBufferSizeMB( determineGoodBufferSize( writerConfig.getRAMBufferSizeMB() ) ); return new IndexWriter( dir, writerConfig ); } catch ( IOException e ) { IOUtils.closeAllSilently( dir ); throw new RuntimeException( e ); } }
IndexWriterConfig iwc = new IndexWriterConfig(analyzer); iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); iwc.setRAMBufferSizeMB(env.getRamBufferSize());
public static IndexWriterConfig standard( Analyzer analyzer ) { IndexWriterConfig writerConfig = new IndexWriterConfig( analyzer ); writerConfig.setMaxBufferedDocs( MAX_BUFFERED_DOCS ); writerConfig.setMaxBufferedDeleteTerms( MAX_BUFFERED_DELETE_TERMS ); writerConfig.setIndexDeletionPolicy( new SnapshotDeletionPolicy( new KeepOnlyLastCommitDeletionPolicy() ) ); writerConfig.setUseCompoundFile( true ); writerConfig.setRAMBufferSizeMB( STANDARD_RAM_BUFFER_SIZE_MB ); writerConfig.setCodec(new Lucene54Codec() { @Override public PostingsFormat getPostingsFormatForField( String field ) { PostingsFormat postingFormat = super.getPostingsFormatForField( field ); return CODEC_BLOCK_TREE_ORDS_POSTING_FORMAT ? blockTreeOrdsPostingsFormat : postingFormat; } }); if ( CUSTOM_MERGE_SCHEDULER ) { writerConfig.setMergeScheduler( new PooledConcurrentMergeScheduler() ); } LogByteSizeMergePolicy mergePolicy = new LogByteSizeMergePolicy(); mergePolicy.setNoCFSRatio( MERGE_POLICY_NO_CFS_RATIO ); mergePolicy.setMinMergeMB( MERGE_POLICY_MIN_MERGE_MB ); mergePolicy.setMergeFactor( MERGE_POLICY_MERGE_FACTOR ); writerConfig.setMergePolicy( mergePolicy ); return writerConfig; }
private IndexWriter newWriter(final Directory dir, final Analyzer analyzer) throws IOException { final IndexWriterConfig config = new IndexWriterConfig(analyzer); config.setUseCompoundFile(ini.getBoolean("lucene.useCompoundFile", false)); config.setRAMBufferSizeMB(ini.getDouble("lucene.ramBufferSizeMB", IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB)); return new IndexWriter(dir, config); }
public void addCorpora(List<Path> corpora, double ramBufferInMb) throws IOException { Directory dir = FSDirectory.open(indexPath); Analyzer analyzer = CustomAnalyzer.builder() .withTokenizer("standard") .addTokenFilter(LuceneLemmaFilter.Factory.class) .build(); IndexWriterConfig iwc = new IndexWriterConfig(analyzer); iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); iwc.setRAMBufferSizeMB(ramBufferInMb); IndexWriter writer = new IndexWriter(dir, iwc); for (Path path : corpora) { Log.info("Adding %s", path); addDocs(writer, path); } writer.close(); }
public static IndexWriter createIndexWriter(Directory directory, boolean create, AnalyzerType analyzer, double RAMBufferSize) throws IOException { // set the analyzer according to the method params Analyzer tmpAnalyzer = null; if (analyzer == AnalyzerType.SimpleAnalyzer) tmpAnalyzer = new SimpleAnalyzer(); else if (analyzer == AnalyzerType.WhitespaceAnalyzer) tmpAnalyzer = new WhitespaceAnalyzer(); // The config IndexWriterConfig config = new IndexWriterConfig(tmpAnalyzer); if (create) config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); // overwrite if it exists. else config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); // create new if none is there, append otherwise. config.setRAMBufferSizeMB(RAMBufferSize); config.setCodec(new LireCustomCodec()); return new IndexWriter(directory, config); }
public static IndexWriter createIndexWriter(Directory directory, boolean create, AnalyzerType analyzer, double RAMBufferSize) throws IOException { // set the analyzer according to the method params Analyzer tmpAnalyzer = null; if (analyzer == AnalyzerType.SimpleAnalyzer) tmpAnalyzer = new SimpleAnalyzer(); else if (analyzer == AnalyzerType.WhitespaceAnalyzer) tmpAnalyzer = new WhitespaceAnalyzer(); // The config IndexWriterConfig config = new IndexWriterConfig(tmpAnalyzer); if (create) config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); // overwrite if it exists. else config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); // create new if none is there, append otherwise. config.setRAMBufferSizeMB(RAMBufferSize); config.setCodec(new LireCustomCodec()); return new IndexWriter(directory, config); }
config.setRAMBufferSizeMB(512); config.setCommitOnClose(true); if (create)
config.setRAMBufferSizeMB(512); config.setCommitOnClose(true); if (create)
@Override public void applySetting(IndexWriterConfig writerConfig, int value) { writerConfig.setRAMBufferSizeMB( value ); } },
@Override public void applySetting(IndexWriterConfig writerConfig, int value) { writerConfig.setRAMBufferSizeMB( value ); } },
iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac()); iwc.setCodec(engineConfig.getCodec()); iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh
private void initConfigValues() { config.setOpenMode(OpenMode.CREATE); // Optional: for better indexing performance, increase the max heap size to the JVM (eg add -Xmxm or -Xmx1g) config.setRAMBufferSizeMB(WRITER_BUFFER_SIZE); } }
private static IndexWriter createIndexWriter(File indexDir) throws IOException { IndexWriterConfig iwc = new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)); iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE); iwc.setRAMBufferSizeMB(256); FileUtils.deleteDirectory(indexDir); FileUtils.forceMkdir(indexDir); return new IndexWriter(FSDirectory.open(indexDir), iwc); }
static IndexWriter createWriter(String filename) throws IOException { IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Version.LUCENE_48, new StandardAnalyzer(Version.LUCENE_48)); indexWriterConfig.setRAMBufferSizeMB(100); indexWriterConfig.setOpenMode(OpenMode.CREATE); return new IndexWriter(FSDirectory.open(new File("output/"+filename)), indexWriterConfig); }
public static IndexWriterConfig getIndexWriterConfig(Analyzer analyzer, boolean create) { IndexWriterConfig config = new IndexWriterConfig(analyzer); config.setOpenMode(create ? OpenMode.CREATE : OpenMode.CREATE_OR_APPEND); config.setRAMBufferSizeMB(150); // faster indexing // Set merge factor (if using LogMergePolicy, which is the default up to version LUCENE_32, // so yes) MergePolicy mp = config.getMergePolicy(); if (mp instanceof LogMergePolicy) { ((LogMergePolicy) mp).setMergeFactor(40); // faster indexing } return config; }
public static IndexWriterConfig defaultConfig() { final IndexWriterConfig config = new IndexWriterConfig( Version.LUCENE_46, new NexusAnalyzer() ); // default open mode is CreateOrAppend which suits us config.setRAMBufferSizeMB( 2.0 ); // old default config.setMergeScheduler( new SerialMergeScheduler() ); // merging serially config.setWriteLockTimeout(IndexWriterConfig.WRITE_LOCK_TIMEOUT); return config; } }
private void openWriter(Language language) { try { IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_36, AnalyzerFactory.getAnalyzer(language)); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); Directory index = getIndex(indexPathToWrite); if(IndexReader.indexExists(index)) config.setOpenMode(IndexWriterConfig.OpenMode.APPEND); config.setRAMBufferSizeMB(BUFFERRAMSIZE); indexer = new Indexer(config, index); } catch (IOException e) { e.printStackTrace(); } }
public static IndexWriterConfig defaultConfig() { final IndexWriterConfig config = new IndexWriterConfig( new NexusAnalyzer() ); // default open mode is CreateOrAppend which suits us config.setRAMBufferSizeMB( 2.0 ); // old default config.setMergeScheduler( new SerialMergeScheduler() ); // merging serially config.setWriteLockTimeout( IndexWriterConfig.WRITE_LOCK_TIMEOUT ); return config; } }