void writeIndexProperties( IndexPackingRequest request, Properties info ) throws IOException { File propertyFile = new File( request.getContext().getIndexDirectoryFile(), IndexingContext.INDEX_PACKER_PROPERTIES_FILE ); File targetPropertyFile = new File( request.getTargetDir(), IndexingContext.INDEX_REMOTE_PROPERTIES_FILE ); info.setProperty( IndexingContext.INDEX_ID, request.getContext().getId() ); try ( OutputStream os = new FileOutputStream( propertyFile ) ) { info.store( os, null ); } try ( OutputStream os = new FileOutputStream( targetPropertyFile ) ) { info.store( os, null ); } if ( request.isCreateChecksumFiles() ) { FileUtils.fileWrite( new File( targetPropertyFile.getParentFile(), targetPropertyFile.getName() + ".sha1" ).getAbsolutePath(), DigesterUtils.getSha1Digest( targetPropertyFile ) ); FileUtils.fileWrite( new File( targetPropertyFile.getParentFile(), targetPropertyFile.getName() + ".md5" ).getAbsolutePath(), DigesterUtils.getMd5Digest( targetPropertyFile ) ); } }
void writeIndexData( IndexPackingRequest request, List<Integer> docIndexes, File targetArchive ) throws IOException { if ( targetArchive.exists() ) { targetArchive.delete(); } try ( OutputStream os = new FileOutputStream( targetArchive ) ) { IndexDataWriter dw = new IndexDataWriter( os ); dw.write( request.getContext(), request.getIndexReader(), docIndexes ); os.flush(); } }
if ( i >= ( request.getMaxIndexChunks() - 1 ) )
private Properties readIndexProperties( IndexPackingRequest request ) throws IOException { File file = null; if ( request.isUseTargetProperties() || request.getContext().getIndexDirectoryFile() == null ) { file = new File( request.getTargetDir(), IndexingContext.INDEX_REMOTE_PROPERTIES_FILE ); } else { file = new File( request.getContext().getIndexDirectoryFile(), IndexingContext.INDEX_PACKER_PROPERTIES_FILE ); } Properties properties = new Properties(); FileInputStream fos = null; try { fos = new FileInputStream( file ); properties.load( fos ); } finally { if ( fos != null ) { fos.close(); } } return properties; }
if ( request.getTargetDir() == null ) if ( request.getTargetDir().exists() ) if ( !request.getTargetDir().isDirectory() ) request.getTargetDir().getAbsolutePath() ) ); if ( !request.getTargetDir().canWrite() ) request.getTargetDir().getAbsolutePath() ) ); if ( !request.getTargetDir().mkdirs() ) throw new IllegalArgumentException( "Can't create " + request.getTargetDir().getAbsolutePath() ); File v1File = new File( request.getTargetDir(), IndexingContext.INDEX_FILE_PREFIX + ".gz" ); if ( request.isCreateIncrementalChunks() ) File file = new File( request.getTargetDir(), // IndexingContext.INDEX_FILE_PREFIX + "." + info.getProperty( IndexingContext.INDEX_CHUNK_COUNTER ) + ".gz" ); if ( request.isCreateChecksumFiles() ) Date timestamp = request.getContext().getTimestamp(); if ( request.getFormats().contains( IndexPackingRequest.IndexFormat.FORMAT_V1 ) )
new IndexPackingRequest( context, indexSearcher.getIndexReader(), outputFolder ); request.setCreateChecksumFiles( createChecksums ); request.setCreateIncrementalChunks( createIncrementalChunks ); request.setFormats( Arrays.asList( IndexFormat.FORMAT_V1 ) ); request.setMaxIndexChunks( chunkCount.intValue() );
IndexPackingRequest packReq = new IndexPackingRequest(context, targetDir); packReq.setCreateIncrementalChunks(true); packReq.setFormats(Arrays.asList(IndexFormat.FORMAT_V1)); indexPacker.packIndex(packReq);
@Override public void pack( final ArchivaIndexingContext context ) throws IndexUpdateFailedException { executeUpdateFunction( context, indexingContext -> { try { IndexPackingRequest request = new IndexPackingRequest( indexingContext, indexingContext.acquireIndexSearcher( ).getIndexReader( ), indexingContext.getIndexDirectoryFile( ) ); indexPacker.packIndex( request ); indexingContext.updateTimestamp( true ); } catch ( IOException e ) { log.error( "IOException while packing index of context " + context.getId( ) + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ) ); throw new IndexUpdateFailedException( "IOException during update of " + context.getId( ), e ); } } ); }
private List<Integer> getIndexChunk( IndexPackingRequest request, Date timestamp ) throws IOException { final List<Integer> chunk = new ArrayList<>(); final IndexReader r = request.getIndexReader(); Bits liveDocs = MultiFields.getLiveDocs( r ); for ( int i = 0; i < r.maxDoc(); i++ ) { if ( liveDocs == null || liveDocs.get( i ) ) { Document d = r.document( i ); String lastModified = d.get( ArtifactInfo.LAST_MODIFIED ); if ( lastModified != null ) { Date t = new Date( Long.parseLong( lastModified ) ); // Only add documents that were added after the last time we indexed if ( t.after( timestamp ) ) { chunk.add( i ); } } } } return chunk; }
private void cleanUpIncrementalChunks( IndexPackingRequest request, Properties properties ) throws IOException File[] files = request.getTargetDir().listFiles( new FilenameFilter()
if ( request.getTargetDir() == null ) if ( request.getTargetDir().exists() ) if ( !request.getTargetDir().isDirectory() ) request.getTargetDir().getAbsolutePath() ) ); if ( !request.getTargetDir().canWrite() ) request.getTargetDir().getAbsolutePath() ) ); if ( !request.getTargetDir().mkdirs() ) throw new IllegalArgumentException( "Can't create " + request.getTargetDir().getAbsolutePath() ); File v1File = new File( request.getTargetDir(), IndexingContext.INDEX_FILE_PREFIX + ".gz" ); if ( request.isCreateIncrementalChunks() ) File file = new File( request.getTargetDir(), // IndexingContext.INDEX_FILE_PREFIX + "." + info.getProperty( IndexingContext.INDEX_CHUNK_COUNTER ) + ".gz" ); if ( request.isCreateChecksumFiles() ) Date timestamp = request.getContext().getTimestamp(); if ( request.getFormats().contains( IndexPackingRequest.IndexFormat.FORMAT_V1 ) )
private Properties readIndexProperties( IndexPackingRequest request ) throws IOException { File file = null; if ( request.isUseTargetProperties() || request.getContext().getIndexDirectoryFile() == null ) { file = new File( request.getTargetDir(), IndexingContext.INDEX_REMOTE_PROPERTIES_FILE ); } else { file = new File( request.getContext().getIndexDirectoryFile(), IndexingContext.INDEX_PACKER_PROPERTIES_FILE ); } Properties properties = new Properties(); FileInputStream fos = null; try { fos = new FileInputStream( file ); properties.load( fos ); } finally { if ( fos != null ) { fos.close(); } } return properties; }
IndexPackingRequest request = new IndexPackingRequest( context, // context.acquireIndexSearcher( ).getIndexReader( ),
private List<Integer> getIndexChunk( IndexPackingRequest request, Date timestamp ) throws IOException { final List<Integer> chunk = new ArrayList<>(); final IndexReader r = request.getIndexReader(); Bits liveDocs = MultiFields.getLiveDocs( r ); for ( int i = 0; i < r.maxDoc(); i++ ) { if ( liveDocs == null || liveDocs.get( i ) ) { Document d = r.document( i ); String lastModified = d.get( ArtifactInfo.LAST_MODIFIED ); if ( lastModified != null ) { Date t = new Date( Long.parseLong( lastModified ) ); // Only add documents that were added after the last time we indexed if ( t.after( timestamp ) ) { chunk.add( i ); } } } } return chunk; }
private void cleanUpIncrementalChunks( IndexPackingRequest request, Properties properties ) throws IOException File[] files = request.getTargetDir().listFiles( new FilenameFilter()
if ( request.getTargetDir() == null ) if ( request.getTargetDir().exists() ) if ( !request.getTargetDir().isDirectory() ) request.getTargetDir().getAbsolutePath() ) ); if ( !request.getTargetDir().canWrite() ) request.getTargetDir().getAbsolutePath() ) ); if ( !request.getTargetDir().mkdirs() ) throw new IllegalArgumentException( "Can't create " + request.getTargetDir().getAbsolutePath() ); File v1File = new File( request.getTargetDir(), IndexingContext.INDEX_FILE_PREFIX + ".gz" ); if ( request.isCreateIncrementalChunks() ) File file = new File( request.getTargetDir(), // IndexingContext.INDEX_FILE_PREFIX + "." + info.getProperty( IndexingContext.INDEX_CHUNK_COUNTER ) + ".gz" ); if ( request.isCreateChecksumFiles() ) Date timestamp = request.getContext().getTimestamp(); if ( request.getFormats().contains( IndexPackingRequest.IndexFormat.FORMAT_V1 ) )
void writeIndexProperties( IndexPackingRequest request, Properties info ) throws IOException { File propertyFile = new File( request.getContext().getIndexDirectoryFile(), IndexingContext.INDEX_PACKER_PROPERTIES_FILE ); File targetPropertyFile = new File( request.getTargetDir(), IndexingContext.INDEX_REMOTE_PROPERTIES_FILE ); info.setProperty( IndexingContext.INDEX_ID, request.getContext().getId() ); try ( OutputStream os = new FileOutputStream( propertyFile ) ) { info.store( os, null ); } try ( OutputStream os = new FileOutputStream( targetPropertyFile ) ) { info.store( os, null ); } if ( request.isCreateChecksumFiles() ) { FileUtils.fileWrite( new File( targetPropertyFile.getParentFile(), targetPropertyFile.getName() + ".sha1" ).getAbsolutePath(), DigesterUtils.getSha1Digest( targetPropertyFile ) ); FileUtils.fileWrite( new File( targetPropertyFile.getParentFile(), targetPropertyFile.getName() + ".md5" ).getAbsolutePath(), DigesterUtils.getMd5Digest( targetPropertyFile ) ); } }
private Properties readIndexProperties( IndexPackingRequest request ) throws IOException { File file = null; if ( request.isUseTargetProperties() || request.getContext().getIndexDirectoryFile() == null ) { file = new File( request.getTargetDir(), IndexingContext.INDEX_REMOTE_PROPERTIES_FILE ); } else { file = new File( request.getContext().getIndexDirectoryFile(), IndexingContext.INDEX_PACKER_PROPERTIES_FILE ); } Properties properties = new Properties(); FileInputStream fos = null; try { fos = new FileInputStream( file ); properties.load( fos ); } finally { if ( fos != null ) { fos.close(); } } return properties; }
IndexPackingRequest request = new IndexPackingRequest( mergedCtx, //
void writeIndexData( IndexPackingRequest request, List<Integer> docIndexes, File targetArchive ) throws IOException { if ( targetArchive.exists() ) { targetArchive.delete(); } try ( OutputStream os = new FileOutputStream( targetArchive ) ) { IndexDataWriter dw = new IndexDataWriter( os ); dw.write( request.getContext(), request.getIndexReader(), docIndexes ); os.flush(); } }