/** * Snapshots the last commit. Once this method returns, the * snapshot information is persisted in the directory. * * @see SnapshotDeletionPolicy#snapshot */ @Override public synchronized IndexCommit snapshot() throws IOException { IndexCommit ic = super.snapshot(); boolean success = false; try { persist(); success = true; } finally { if (!success) { try { super.release(ic); } catch (Exception e) { // Suppress so we keep throwing original exception } } } return ic; }
private IndexWriter newIndexWriter( IndexIdentifier identifier ) throws ExplicitIndexNotFoundKernelException { try { Directory indexDirectory = getIndexDirectory( identifier ); IndexType type = getType( identifier ); IndexWriterConfig writerConfig = new IndexWriterConfig( type.analyzer ); writerConfig.setIndexDeletionPolicy( new SnapshotDeletionPolicy( new KeepOnlyLastCommitDeletionPolicy() ) ); Similarity similarity = type.getSimilarity(); if ( similarity != null ) { writerConfig.setSimilarity( similarity ); } return new IndexWriter( indexDirectory, writerConfig ); } catch ( IOException e ) { throw new RuntimeException( e ); } } }
@Override public synchronized void onCommit(List<? extends IndexCommit> commits) throws IOException { primary.onCommit(wrapCommits(commits)); lastCommit = commits.get(commits.size() - 1); }
@Override public void close() { try { snapshotDeletionPolicy.release( getIndexCommit() ); } catch ( IOException e ) { throw new SnapshotReleaseException( "Unable to release lucene index snapshot for index in: " + getIndexDirectory(), e ); } }
WritableIndexSnapshotFileIterator( File indexDirectory, SnapshotDeletionPolicy snapshotDeletionPolicy ) throws IOException { super( indexDirectory, snapshotDeletionPolicy.snapshot() ); this.snapshotDeletionPolicy = snapshotDeletionPolicy; }
/** * Release a snapshotted commit. * * @param commit * the commit previously returned by {@link #snapshot} */ public synchronized void release(IndexCommit commit) throws IOException { long gen = commit.getGeneration(); releaseGen(gen); }
/** * Snapshots the last commit and returns it. Once a commit is 'snapshotted,' it is protected * from deletion (as long as this {@link IndexDeletionPolicy} is used). The * snapshot can be removed by calling {@link #release(IndexCommit)} followed * by a call to {@link IndexWriter#deleteUnusedFiles()}. * * <p> * <b>NOTE:</b> while the snapshot is held, the files it references will not * be deleted, which will consume additional disk space in your index. If you * take a snapshot at a particularly bad time (say just before you call * forceMerge) then in the worst case this could consume an extra 1X of your * total index size, until you release the snapshot. * * @throws IllegalStateException * if this index does not have any commits yet * @return the {@link IndexCommit} that was snapshotted. */ public synchronized IndexCommit snapshot() throws IOException { if (!initCalled) { throw new IllegalStateException("this instance is not being used by IndexWriter; be sure to use the instance returned from writer.getConfig().getIndexDeletionPolicy()"); } if (lastCommit == null) { // No commit yet, eg this is a new IndexWriter: throw new IllegalStateException("No index commit to snapshot"); } incRef(lastCommit); return lastCommit; }
@Override public void close() { RuntimeException exception = null; for ( Pair<SnapshotDeletionPolicy,IndexCommit> policyAndCommit : snapshots ) { try { policyAndCommit.first().release( policyAndCommit.other() ); } catch ( IOException | RuntimeException e ) { if ( exception == null ) { exception = e instanceof IOException ? new UncheckedIOException( (IOException) e ) : (RuntimeException) e; } else { exception.addSuppressed( e ); } } } if ( exception != null ) { throw exception; } } };
/** * Deletes a snapshotted commit by generation. Once this method returns, the snapshot * information is persisted in the directory. * * @see IndexCommit#getGeneration * @see SnapshotDeletionPolicy#release */ public synchronized void release(long gen) throws IOException { super.releaseGen(gen); persist(); }
/** * Snapshots the last commit and returns it. Once a commit is 'snapshotted,' it is protected * from deletion (as long as this {@link IndexDeletionPolicy} is used). The * snapshot can be removed by calling {@link #release(IndexCommit)} followed * by a call to {@link IndexWriter#deleteUnusedFiles()}. * * <p> * <b>NOTE:</b> while the snapshot is held, the files it references will not * be deleted, which will consume additional disk space in your index. If you * take a snapshot at a particularly bad time (say just before you call * forceMerge) then in the worst case this could consume an extra 1X of your * total index size, until you release the snapshot. * * @throws IllegalStateException * if this index does not have any commits yet * @return the {@link IndexCommit} that was snapshotted. */ public synchronized IndexCommit snapshot() throws IOException { if (!initCalled) { throw new IllegalStateException("this instance is not being used by IndexWriter; be sure to use the instance returned from writer.getConfig().getIndexDeletionPolicy()"); } if (lastCommit == null) { // No commit yet, eg this is a new IndexWriter: throw new IllegalStateException("No index commit to snapshot"); } incRef(lastCommit); return lastCommit; }
/** * Snapshots the last commit. Once this method returns, the * snapshot information is persisted in the directory. * * @see SnapshotDeletionPolicy#snapshot */ @Override public synchronized IndexCommit snapshot() throws IOException { IndexCommit ic = super.snapshot(); boolean success = false; try { persist(); success = true; } finally { if (!success) { try { super.release(ic); } catch (Exception e) { // Suppress so we keep throwing original exception } } } return ic; }
/** * Deletes a snapshotted commit. Once this method returns, the snapshot * information is persisted in the directory. * * @see SnapshotDeletionPolicy#release */ @Override public synchronized void release(IndexCommit commit) throws IOException { super.release(commit); boolean success = false; try { persist(); success = true; } finally { if (!success) { try { incRef(commit); } catch (Exception e) { // Suppress so we keep throwing original exception } } } }
FromCommit(final Path directoryPath, final SnapshotDeletionPolicy indexSnapshots) throws IOException { this(directoryPath, indexSnapshots, indexSnapshots.snapshot()); }
public static IndexWriterConfig standard( Analyzer analyzer ) { IndexWriterConfig writerConfig = new IndexWriterConfig( analyzer ); writerConfig.setMaxBufferedDocs( MAX_BUFFERED_DOCS ); writerConfig.setMaxBufferedDeleteTerms( MAX_BUFFERED_DELETE_TERMS ); writerConfig.setIndexDeletionPolicy( new SnapshotDeletionPolicy( new KeepOnlyLastCommitDeletionPolicy() ) ); writerConfig.setUseCompoundFile( true ); writerConfig.setRAMBufferSizeMB( STANDARD_RAM_BUFFER_SIZE_MB ); writerConfig.setCodec(new Lucene54Codec() { @Override public PostingsFormat getPostingsFormatForField( String field ) { PostingsFormat postingFormat = super.getPostingsFormatForField( field ); return CODEC_BLOCK_TREE_ORDS_POSTING_FORMAT ? blockTreeOrdsPostingsFormat : postingFormat; } }); if ( CUSTOM_MERGE_SCHEDULER ) { writerConfig.setMergeScheduler( new PooledConcurrentMergeScheduler() ); } LogByteSizeMergePolicy mergePolicy = new LogByteSizeMergePolicy(); mergePolicy.setNoCFSRatio( MERGE_POLICY_NO_CFS_RATIO ); mergePolicy.setMinMergeMB( MERGE_POLICY_MIN_MERGE_MB ); mergePolicy.setMergeFactor( MERGE_POLICY_MERGE_FACTOR ); writerConfig.setMergePolicy( mergePolicy ); return writerConfig; }
@Override public synchronized void onInit(List<? extends IndexCommit> commits) throws IOException { initCalled = true; primary.onInit(wrapCommits(commits)); for(IndexCommit commit : commits) { if (refCounts.containsKey(commit.getGeneration())) { indexCommits.put(commit.getGeneration(), commit); } } if (!commits.isEmpty()) { lastCommit = commits.get(commits.size() - 1); } }
/** * Deletes a snapshotted commit by generation. Once this method returns, the snapshot * information is persisted in the directory. * * @see IndexCommit#getGeneration * @see SnapshotDeletionPolicy#release */ public synchronized void release(long gen) throws IOException { super.releaseGen(gen); persist(); }
/** * Snapshots the last commit and returns it. Once a commit is 'snapshotted,' it is protected * from deletion (as long as this {@link IndexDeletionPolicy} is used). The * snapshot can be removed by calling {@link #release(IndexCommit)} followed * by a call to {@link IndexWriter#deleteUnusedFiles()}. * * <p> * <b>NOTE:</b> while the snapshot is held, the files it references will not * be deleted, which will consume additional disk space in your index. If you * take a snapshot at a particularly bad time (say just before you call * forceMerge) then in the worst case this could consume an extra 1X of your * total index size, until you release the snapshot. * * @throws IllegalStateException * if this index does not have any commits yet * @return the {@link IndexCommit} that was snapshotted. */ public synchronized IndexCommit snapshot() throws IOException { if (!initCalled) { throw new IllegalStateException("this instance is not being used by IndexWriter; be sure to use the instance returned from writer.getConfig().getIndexDeletionPolicy()"); } if (lastCommit == null) { // No commit yet, eg this is a new IndexWriter: throw new IllegalStateException("No index commit to snapshot"); } incRef(lastCommit); return lastCommit; }
/** * Snapshots the last commit. Once this method returns, the * snapshot information is persisted in the directory. * * @see SnapshotDeletionPolicy#snapshot */ @Override public synchronized IndexCommit snapshot() throws IOException { IndexCommit ic = super.snapshot(); boolean success = false; try { persist(); success = true; } finally { if (!success) { try { super.release(ic); } catch (Exception e) { // Suppress so we keep throwing original exception } } } return ic; }
/** * Releases a snapshot taken from {@link #acquireIndexCommit(boolean)} this must be called to release the resources * referenced by the given snapshot {@link IndexCommit}. */ public void releaseIndexCommit(IndexCommit snapshot) throws IOException { deletionPolicy.release(snapshot); }