public static IndexWriterConfig standard( Analyzer analyzer ) { IndexWriterConfig writerConfig = new IndexWriterConfig( analyzer ); writerConfig.setMaxBufferedDocs( MAX_BUFFERED_DOCS ); writerConfig.setMaxBufferedDeleteTerms( MAX_BUFFERED_DELETE_TERMS ); writerConfig.setIndexDeletionPolicy( new SnapshotDeletionPolicy( new KeepOnlyLastCommitDeletionPolicy() ) ); writerConfig.setUseCompoundFile( true ); writerConfig.setRAMBufferSizeMB( STANDARD_RAM_BUFFER_SIZE_MB ); writerConfig.setCodec(new Lucene54Codec() { @Override public PostingsFormat getPostingsFormatForField( String field ) { PostingsFormat postingFormat = super.getPostingsFormatForField( field ); return CODEC_BLOCK_TREE_ORDS_POSTING_FORMAT ? blockTreeOrdsPostingsFormat : postingFormat; } }); if ( CUSTOM_MERGE_SCHEDULER ) { writerConfig.setMergeScheduler( new PooledConcurrentMergeScheduler() ); } LogByteSizeMergePolicy mergePolicy = new LogByteSizeMergePolicy(); mergePolicy.setNoCFSRatio( MERGE_POLICY_NO_CFS_RATIO ); mergePolicy.setMinMergeMB( MERGE_POLICY_MIN_MERGE_MB ); mergePolicy.setMergeFactor( MERGE_POLICY_MERGE_FACTOR ); writerConfig.setMergePolicy( mergePolicy ); return writerConfig; }
private IndexWriter newWriter(final Directory dir, final Analyzer analyzer) throws IOException { final IndexWriterConfig config = new IndexWriterConfig(analyzer); config.setUseCompoundFile(ini.getBoolean("lucene.useCompoundFile", false)); config.setRAMBufferSizeMB(ini.getDouble("lucene.ramBufferSizeMB", IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB)); return new IndexWriter(dir, config); }
iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac()); iwc.setCodec(engineConfig.getCodec()); iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh if (config().getIndexSort() != null) { iwc.setIndexSort(config().getIndexSort());
public IndexWriter openLuceneIndex(File aIndexDir) throws IOException { Directory directory = FSDirectory.open(aIndexDir.toPath()); // Create parser configuration as a JSON object JSONObject jsonParserConfiguration = new JSONObject(); // Add the project id to the configuration jsonParserConfiguration.put("projectId", project.getId()); // Tokenizer parameters Map<String, String> paramsTokenizer = new HashMap<String, String>(); paramsTokenizer.put(MtasTokenizerFactory.ARGUMENT_PARSER, MTAS_PARSER); paramsTokenizer.put(MtasTokenizerFactory.ARGUMENT_PARSER_ARGS, jsonParserConfiguration.toString()); // Build analyzer Analyzer mtasAnalyzer = CustomAnalyzer.builder() .withTokenizer(MTAS_TOKENIZER, paramsTokenizer).build(); Map<String, Analyzer> analyzerPerField = new HashMap<String, Analyzer>(); analyzerPerField.put(FIELD_CONTENT, mtasAnalyzer); PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new StandardAnalyzer(), analyzerPerField); // Build IndexWriter IndexWriterConfig config = new IndexWriterConfig(analyzer); config.setUseCompoundFile(false); config.setCodec(Codec.forName(MtasCodec.MTAS_CODEC_NAME)); return new IndexWriter(directory, config); }
config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); config.setRAMBufferSizeMB(args.memorybufferSize); config.setUseCompoundFile(false); config.setMergeScheduler(new ConcurrentMergeScheduler());
.setOpenMode(OpenMode.CREATE).setUseCompoundFile(false)
public IndexWriterConfig createIndexWriterConfig(ODocument metadata, Analyzer analyzer) { IndexWriterConfig config = new IndexWriterConfig(analyzer); config.setOpenMode(CREATE_OR_APPEND); if (metadata.containsField("use_compound_file")) config.setUseCompoundFile(metadata.<Boolean>field("use_compound_file")); if (metadata.containsField("ram_buffer_MB")) config.setRAMBufferSizeMB(Double.valueOf(metadata.<String>field("ram_buffer_MB"))); if (metadata.containsField("max_buffered_docs")) config.setMaxBufferedDocs(Integer.valueOf(metadata.<String>field("max_buffered_docs"))); // TODO REMOVED // if (metadata.containsField("max_buffered_delete_terms")) // config.setMaxBufferedDeleteTerms(Integer.valueOf(metadata.<String>field("max_buffered_delete_terms"))); if (metadata.containsField("ram_per_thread_MB")) config.setRAMPerThreadHardLimitMB(Integer.valueOf(metadata.<String>field("ram_per_thread_MB"))); return config; } }
.setOpenMode(OpenMode.CREATE).setUseCompoundFile(false)
.setOpenMode(OpenMode.CREATE).setUseCompoundFile(false)
.setOpenMode(OpenMode.CREATE).setUseCompoundFile(false)
"org.apache.lucene.index.LogByteSizeMergePolicy"); boolean isCompound = config.get("compound", true); iwConf.setUseCompoundFile(isCompound); if (mergePolicy.equals(NoMergePolicy.class.getName())) { iwConf.setMergePolicy(NoMergePolicy.INSTANCE);
IndexWriter createWriter(boolean create) throws IOException { try { final IndexWriterConfig iwc = new IndexWriterConfig(engineConfig.getAnalyzer()); iwc.setCommitOnClose(false); // we by default don't commit on close iwc.setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND); iwc.setIndexDeletionPolicy(deletionPolicy); // with tests.verbose, lucene sets this up: plumb to align with filesystem stream boolean verbose = false; try { verbose = Boolean.parseBoolean(System.getProperty("tests.verbose")); } catch (Exception ignore) { } iwc.setInfoStream(verbose ? InfoStream.getDefault() : new LoggerInfoStream(logger)); iwc.setMergeScheduler(mergeScheduler); MergePolicy mergePolicy = config().getMergePolicy(); // Give us the opportunity to upgrade old segments while performing // background merges mergePolicy = new ElasticsearchMergePolicy(mergePolicy); iwc.setMergePolicy(mergePolicy); iwc.setSimilarity(engineConfig.getSimilarity()); iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac()); iwc.setCodec(engineConfig.getCodec()); iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh return new IndexWriter(store.directory(), iwc); } catch (LockObtainFailedException ex) { logger.warn("could not lock IndexWriter", ex); throw ex; } }
IndexWriter createWriter(boolean create) throws IOException { try { final IndexWriterConfig iwc = new IndexWriterConfig(engineConfig.getAnalyzer()); iwc.setCommitOnClose(false); // we by default don't commit on close iwc.setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND); iwc.setIndexDeletionPolicy(deletionPolicy); // with tests.verbose, lucene sets this up: plumb to align with filesystem stream boolean verbose = false; try { verbose = Boolean.parseBoolean(System.getProperty("tests.verbose")); } catch (Exception ignore) { } iwc.setInfoStream(verbose ? InfoStream.getDefault() : new LoggerInfoStream(logger)); iwc.setMergeScheduler(mergeScheduler); MergePolicy mergePolicy = config().getMergePolicy(); // Give us the opportunity to upgrade old segments while performing // background merges mergePolicy = new ElasticsearchMergePolicy(mergePolicy); iwc.setMergePolicy(mergePolicy); iwc.setSimilarity(engineConfig.getSimilarity()); iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac()); iwc.setCodec(engineConfig.getCodec()); iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh return new IndexWriter(store.directory(), iwc); } catch (LockObtainFailedException ex) { logger.warn("could not lock IndexWriter", ex); throw ex; } }
private IndexWriterConfig getIndexWriterConfig() { final IndexWriterConfig iwc = new IndexWriterConfig(engineConfig.getAnalyzer()); iwc.setCommitOnClose(false); // we by default don't commit on close iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND); iwc.setIndexDeletionPolicy(combinedDeletionPolicy); // with tests.verbose, lucene sets this up: plumb to align with filesystem stream boolean verbose = false; try { verbose = Boolean.parseBoolean(System.getProperty("tests.verbose")); } catch (Exception ignore) { } iwc.setInfoStream(verbose ? InfoStream.getDefault() : new LoggerInfoStream(logger)); iwc.setMergeScheduler(mergeScheduler); MergePolicy mergePolicy = config().getMergePolicy(); // Give us the opportunity to upgrade old segments while performing // background merges mergePolicy = new ElasticsearchMergePolicy(mergePolicy); iwc.setMergePolicy(mergePolicy); iwc.setSimilarity(engineConfig.getSimilarity()); iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac()); iwc.setCodec(engineConfig.getCodec()); iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh if (config().getIndexSort() != null) { iwc.setIndexSort(config().getIndexSort()); } return iwc; }
config.setUseCompoundFile(false);
c.setMergedSegmentWarmer(new SimpleMergedSegmentWarmer(c.getInfoStream())); c.setUseCompoundFile(r.nextBoolean()); c.setReaderPooling(r.nextBoolean()); return c;
iwc.setUseCompoundFile(this.engineConfig.isCompoundOnFlush());
iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac()); iwc.setCodec(engineConfig.getCodec()); iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh if (config().getIndexSort() != null) { iwc.setIndexSort(config().getIndexSort());
indexWriterConfig.setRAMBufferSizeMB(settings.ramBufferSize); if (settings.useCompoundFile != null) indexWriterConfig.setUseCompoundFile(settings.useCompoundFile); if(settings.useSimpleTextCodec != null && settings.useSimpleTextCodec) indexWriterConfig.setCodec(new SimpleTextCodec());