private void processEvents(boolean triggerMerge) throws IOException { if (tragedy.get() == null) { Event event; while ((event = eventQueue.poll()) != null) { event.process(this); } } if (triggerMerge) { maybeMerge(getConfig().getMergePolicy(), MergeTrigger.SEGMENT_FLUSH, UNBOUNDED_MAX_MERGE_SEGMENTS); } }
/** * Create index snapshot iterator for a writable index. * @param indexFolder index location folder * @param indexWriter index writer * @return index file name iterator * @throws IOException */ public static ResourceIterator<File> forIndex( File indexFolder, IndexWriter indexWriter ) throws IOException { IndexDeletionPolicy deletionPolicy = indexWriter.getConfig().getIndexDeletionPolicy(); if ( deletionPolicy instanceof SnapshotDeletionPolicy ) { SnapshotDeletionPolicy policy = (SnapshotDeletionPolicy) deletionPolicy; return hasCommits( indexWriter ) ? new WritableIndexSnapshotFileIterator( indexFolder, policy ) : emptyResourceIterator(); } else { throw new UnsupportedIndexDeletionPolicy( "Can't perform index snapshot with specified index deletion " + "policy: " + deletionPolicy.getClass().getName() + ". " + "Only " + SnapshotDeletionPolicy.class.getName() + " is " + "supported" ); } }
if (writer.getConfig().getOpenMode() == OpenMode.CREATE) { writer.addDocument(doc); } else {
for ( IndexReference index : getAllIndexes() ) SnapshotDeletionPolicy deletionPolicy = (SnapshotDeletionPolicy) index.getWriter().getConfig() .getIndexDeletionPolicy(); File indexDirectory = getFileDirectory( baseStorePath, index.getIdentifier() );
try { final SegmentReader reader = rld.getReadOnlyClone(IOContext.READ); if (reader.numDocs() > 0 || writer.getConfig().mergePolicy.keepFullyDeletedSegment(() -> reader)) {
/** Close segment states previously opened with openSegmentStates. */ public static BufferedUpdatesStream.ApplyDeletesResult closeSegmentStates(IndexWriter writer, BufferedUpdatesStream.SegmentState[] segStates, boolean success) throws IOException { List<SegmentCommitInfo> allDeleted = null; long totDelCount = 0; final List<BufferedUpdatesStream.SegmentState> segmentStates = Arrays.asList(segStates); for (BufferedUpdatesStream.SegmentState segState : segmentStates) { if (success) { totDelCount += segState.rld.getDelCount() - segState.startDelCount; int fullDelCount = segState.rld.getDelCount(); assert fullDelCount <= segState.rld.info.info.maxDoc() : fullDelCount + " > " + segState.rld.info.info.maxDoc(); if (segState.rld.isFullyDeleted() && writer.getConfig().getMergePolicy().keepFullyDeletedSegment(() -> segState.reader) == false) { if (allDeleted == null) { allDeleted = new ArrayList<>(); } allDeleted.add(segState.reader.getOriginalSegmentInfo()); } } } IOUtils.close(segmentStates); if (writer.infoStream.isEnabled("BD")) { writer.infoStream.message("BD", "closeSegmentStates: " + totDelCount + " new deleted documents; pool " + writer.getPendingUpdatesCount()+ " packets; bytesUsed=" + writer.getReaderPoolRamBytesUsed()); } return new BufferedUpdatesStream.ApplyDeletesResult(totDelCount > 0, allDeleted); }
LiveIndexWriterConfig getCurrentIndexWriterConfig() { return indexWriter.getConfig(); }
if (writer.getConfig().getOpenMode() == OpenMode.CREATE)
indexWriterConfig.setMergePolicy(logDocMergePolicy); w = new IndexWriter(index, indexWriterConfig); w.getConfig().setRAMBufferSizeMB(32);
w = new IndexWriter(index, indexWriterConfig); w.getConfig().setRAMBufferSizeMB(32);
public WithIndex(final String masterUuid, final Path resourcesPath, final Path indexDirectoryPath, final IndexWriter indexWriter, final Path metadataDirectory, final String... metadataItems) { super(masterUuid, resourcesPath, metadataDirectory, metadataItems); this.indexDirectoryPath = indexDirectoryPath; this.indexSnapshot = (SnapshotDeletionPolicy) indexWriter.getConfig().getIndexDeletionPolicy(); this.indexSessions = new HashMap<>(); }
private void addDocument(final File file, final Document doc) throws IOException { if (indexWriter.getConfig().getOpenMode() == OpenMode.CREATE) { indexWriter.addDocument(doc); } else { Term term = termFactory.createTerm(fieldName, file.getPath()); indexWriter.updateDocument(term, doc); } }
private void processEvents(boolean triggerMerge) throws IOException { if (tragedy.get() == null) { Event event; while ((event = eventQueue.poll()) != null) { event.process(this); } } if (triggerMerge) { maybeMerge(getConfig().getMergePolicy(), MergeTrigger.SEGMENT_FLUSH, UNBOUNDED_MAX_MERGE_SEGMENTS); } }
private void configureIndexWriter(final IndexWriter indexWriter, Configuration configuration) { LiveIndexWriterConfig config = indexWriter.getConfig(); config.setUseCompoundFile(configuration.isCompoundIndexFileFormat()); if (batchMode.get()) { config.setMaxBufferedDocs(configuration.getBatchMaxBufferedDocs()); } else { config.setMaxBufferedDocs(configuration.getInteractiveMaxBufferedDocs()); } }
private void updateIndexWriterSettings() { try { final LiveIndexWriterConfig iwc = indexWriter.getConfig(); iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().mbFrac()); iwc.setUseCompoundFile(engineConfig.isCompoundOnFlush()); } catch (AlreadyClosedException ex) { // ignore } }
@Test public void testIndexFileShouldCreateIndexWhenWriterInCreateMode() throws IOException { //GIVEN given(writer.getConfig().getOpenMode()).willReturn(OpenMode.CREATE); given(file.getPath()).willReturn("path"); given(termFactory.createTerm(FIELD_NAME, "path")).willReturn(term); //WHEN underTest.indexFile(file); //THEN verify(writer).addDocument(document); }
@Test public void indexWriterConfig_Scheduler_Remote() throws Exception{ LuceneIndexDefinition defn = new LuceneIndexDefinition(root, builder.getNodeState(), "/foo"); DefaultIndexWriter writer = createWriter(defn, true); IndexWriter w = writer.getWriter(); assertThat(w.getConfig().getMergeScheduler(), instanceOf(SerialMergeScheduler.class)); }
@Test public void indexWriterConfig_Scheduler_Local() throws Exception{ FSDirectoryFactory fsdir = new FSDirectoryFactory(folder.getRoot()); LuceneIndexDefinition defn = new LuceneIndexDefinition(root, builder.getNodeState(), "/foo"); DefaultIndexWriter writer = new DefaultIndexWriter(defn, builder, fsdir, INDEX_DATA_CHILD_NAME, SUGGEST_DATA_CHILD_NAME, true, writerConfig); IndexWriter w = writer.getWriter(); assertThat(w.getConfig().getMergeScheduler(), instanceOf(ConcurrentMergeScheduler.class)); }
@Test public void configRAMSize() throws Exception{ writerConfig = new LuceneIndexWriterConfig(42); LuceneIndexDefinition defn = new LuceneIndexDefinition(root, builder.getNodeState(), "/foo"); DefaultIndexWriter writer = createWriter(defn, true); IndexWriter w = writer.getWriter(); assertEquals(w.getConfig().getRAMBufferSizeMB(), 42, 0); }