Resource gatherExplicitIndexFiles( Collection<StoreFileMetadata> files ) throws IOException { final Collection<ResourceIterator<File>> snapshots = new ArrayList<>(); for ( IndexImplementation indexProvider : explicitIndexProviders.allIndexProviders() ) { ResourceIterator<File> snapshot = indexProvider.listStoreFiles(); snapshots.add( snapshot ); getSnapshotFilesMetadata( snapshot, files ); } // Intentionally don't close the snapshot here, return it for closing by the consumer of // the targetFiles list. return new MultiResource( snapshots ); }
@Override public TransactionApplier newApplier( String providerName, boolean recovery ) { return provider.getProviderByName( providerName ).newApplier( recovery ); } }
@Override public void flushAndForce( IOLimiter limiter ) { indexingService.forceAll( limiter ); labelScanStore.force( limiter ); for ( IndexImplementation index : explicitIndexProviderLookup.allIndexProviders() ) { index.force(); } neoStores.flush( limiter ); }
@Override public boolean checkIndexExistence( IndexEntityType entityType, String indexName, Map<String,String> config ) { Map<String, String> configuration = indexConfigStore.get( entityType.entityClass(), indexName ); if ( configuration == null ) { return false; } String providerName = configuration.get( IndexManager.PROVIDER ); IndexImplementation provider = providerLookup.getProviderByName( providerName ); assertConfigMatches( provider, indexName, configuration, config ); return true; }
@Test public void shouldCloseIndexAndLabelScanSnapshots() throws Exception { // Given LabelScanStore labelScanStore = mock( LabelScanStore.class ); IndexingService indexingService = mock( IndexingService.class ); ExplicitIndexProvider explicitIndexes = mock( ExplicitIndexProvider.class ); when( explicitIndexes.allIndexProviders() ).thenReturn( Collections.emptyList() ); DatabaseLayout databaseLayout = mock( DatabaseLayout.class ); when( databaseLayout.metadataStore() ).thenReturn( mock( File.class ) ); LogFiles logFiles = mock( LogFiles.class ); filesInStoreDirAre( databaseLayout, STANDARD_STORE_DIR_FILES, STANDARD_STORE_DIR_DIRECTORIES ); StorageEngine storageEngine = mock( StorageEngine.class ); NeoStoreFileListing fileListing = new NeoStoreFileListing( databaseLayout, logFiles, labelScanStore, indexingService, explicitIndexes, storageEngine ); ResourceIterator<File> scanSnapshot = scanStoreFilesAre( labelScanStore, new String[]{"blah/scan.store", "scan.more"} ); ResourceIterator<File> indexSnapshot = indexFilesAre( indexingService, new String[]{"schema/index/my.index"} ); ResourceIterator<StoreFileMetadata> result = fileListing.builder().excludeLogFiles().build(); // When result.close(); // Then verify( scanSnapshot ).close(); verify( indexSnapshot ).close(); }
@Override public ExplicitIndex relationshipChanges( String indexName ) throws ExplicitIndexNotFoundKernelException { Map<String, String> configuration = indexConfigStore.get( Relationship.class, indexName ); if ( configuration == null ) { throw new ExplicitIndexNotFoundKernelException( "Relationship index '" + indexName + " not found" ); } String providerName = configuration.get( IndexManager.PROVIDER ); IndexImplementation provider = providerLookup.getProviderByName( providerName ); ExplicitIndexProviderTransaction transaction = transactions.get( providerName ); if ( transaction == null ) { transactions.put( providerName, transaction = provider.newTransaction( this ) ); } return transaction.relationshipIndex( indexName, configuration ); }
@Test void shouldCountAllIndexFiles() throws Exception { // Explicit index file File explicitIndex = testDirectory.databaseLayout().file( "explicitIndex" ); createFileOfSize( explicitIndex, 1 ); IndexImplementation indexImplementation = mock( IndexImplementation.class ); when( indexImplementation.getIndexImplementationDirectory( any() ) ).thenReturn( explicitIndex ); when( explicitIndexProviderLookup.allIndexProviders() ).thenReturn( iterable( indexImplementation ) ); // Schema index files File schemaIndex = testDirectory.databaseLayout().file( "schemaIndex" ); createFileOfSize( schemaIndex, 2 ); IndexDirectoryStructure directoryStructure = mock( IndexDirectoryStructure.class ); when( directoryStructure.rootDirectory() ).thenReturn( schemaIndex ); when( indexProvider.directoryStructure() ).thenReturn( directoryStructure ); File schemaIndex2 = testDirectory.databaseLayout().file( "schemaIndex2" ); createFileOfSize( schemaIndex2, 3 ); IndexDirectoryStructure directoryStructure2 = mock( IndexDirectoryStructure.class ); when( directoryStructure2.rootDirectory() ).thenReturn( schemaIndex2 ); when( indexProvider2.directoryStructure() ).thenReturn( directoryStructure2 ); // Label scan store File labelScan = testDirectory.databaseLayout().labelScanStore(); createFileOfSize( labelScan, 4 ); when( labelScanStore.getLabelScanStoreFile() ).thenReturn( labelScan ); // Count all files assertEquals( 10, storeSizeBean.getIndexStoreSize() ); }
@Override public ExplicitIndex nodeChanges( String indexName ) throws ExplicitIndexNotFoundKernelException { Map<String, String> configuration = indexConfigStore.get( Node.class, indexName ); if ( configuration == null ) { throw new ExplicitIndexNotFoundKernelException( "Node index '" + indexName + " not found" ); } String providerName = configuration.get( IndexManager.PROVIDER ); IndexImplementation provider = providerLookup.getProviderByName( providerName ); ExplicitIndexProviderTransaction transaction = transactions.computeIfAbsent( providerName, k -> provider.newTransaction( this ) ); return transaction.nodeIndex( indexName, configuration ); }
@Override public long getIndexStoreSize() { long size = 0L; // Add explicit indices for ( IndexImplementation index : explicitIndexProviderLookup.allIndexProviders() ) { size += FileUtils.size( fs, index.getIndexImplementationDirectory( databaseLayout ) ); } // Add schema index MutableLong schemaSize = new MutableLong(); indexProviderMap.accept( provider -> { File rootDirectory = provider.directoryStructure().rootDirectory(); if ( rootDirectory != null ) { schemaSize.add( FileUtils.size( fs, rootDirectory ) ); } // else this provider didn't have any persistent storage } ); size += schemaSize.longValue(); // Add label index size += FileUtils.size( fs, labelScanStore.getLabelScanStoreFile() ); return size; }
private ExplicitIndexProvider getExplicitIndexProvider() { IndexImplementation indexImplementation = mock( IndexImplementation.class ); when( indexImplementation.getIndexImplementationDirectory( storeLayout ) ).thenReturn( originalIndexStore ); when( indexImplementation.getIndexImplementationDirectory( migrationLayout ) ).thenReturn( migratedIndexStore ); ExplicitIndexProvider explicitIndexProvider = mock( ExplicitIndexProvider.class ); when( explicitIndexProvider.getProviderByName( "lucene" ) ).thenReturn( indexImplementation ); return explicitIndexProvider; }
private RecordStorageEngine get( FileSystemAbstraction fs, PageCache pageCache, IndexProvider indexProvider, DatabaseHealth databaseHealth, DatabaseLayout databaseLayout, Function<BatchTransactionApplierFacade, BatchTransactionApplierFacade> transactionApplierTransformer, Monitors monitors, LockService lockService ) { IdGeneratorFactory idGeneratorFactory = new EphemeralIdGenerator.Factory(); ExplicitIndexProvider explicitIndexProviderLookup = mock( ExplicitIndexProvider.class ); when( explicitIndexProviderLookup.allIndexProviders() ).thenReturn( Iterables.empty() ); IndexConfigStore indexConfigStore = new IndexConfigStore( databaseLayout, fs ); JobScheduler scheduler = life.add( createScheduler() ); Config config = Config.defaults( GraphDatabaseSettings.default_schema_provider, indexProvider.getProviderDescriptor().name() ); Dependencies dependencies = new Dependencies(); dependencies.satisfyDependency( indexProvider ); BufferingIdGeneratorFactory bufferingIdGeneratorFactory = new BufferingIdGeneratorFactory( idGeneratorFactory, IdReuseEligibility.ALWAYS, new CommunityIdTypeConfigurationProvider() ); DefaultIndexProviderMap indexProviderMap = new DefaultIndexProviderMap( dependencies, config ); NullLogProvider nullLogProvider = NullLogProvider.getInstance(); life.add( indexProviderMap ); return life.add( new ExtendedRecordStorageEngine( databaseLayout, config, pageCache, fs, nullLogProvider, nullLogProvider, mockedTokenHolders(), mock( SchemaState.class ), new StandardConstraintSemantics(), scheduler, mock( TokenNameLookup.class ), lockService, indexProviderMap, IndexingService.NO_MONITOR, databaseHealth, explicitIndexProviderLookup, indexConfigStore, new SynchronizedArrayIdOrderingQueue(), idGeneratorFactory, new BufferedIdController( bufferingIdGeneratorFactory, scheduler ), transactionApplierTransformer, monitors, RecoveryCleanupWorkCollector.immediate(), OperationalMode.single ) ); }
private ExplicitIndexTransactionStateImpl newExplicitIndexTxState() { indexConfigStore = mock( IndexConfigStore.class ); when( indexConfigStore.get( eq( Node.class ), anyString() ) ).thenReturn( config ); when( indexConfigStore.get( eq( Relationship.class ), anyString() ) ).thenReturn( config ); ExplicitIndexProvider explicitIndexProvider = mock( ExplicitIndexProvider.class ); when( explicitIndexProvider.getProviderByName( anyString() ) ).thenReturn( provider ); return new ExplicitIndexTransactionStateImpl( indexConfigStore, explicitIndexProvider ); } }
dataSourceDependencies.satisfyDependency( watcherService ); life.add( Lifecycles.multiple( explicitIndexProvider.allIndexProviders() ) );
provider = provider == null ? getDefaultProvider( indexName, dbConfig ) : provider; indexProvider = explicitIndexProvider.getProviderByName( provider ); configToUse = indexProvider.fillInDefaults( configToUse ); configToUse = injectDefaultProviderIfMissing( indexName, dbConfig, configToUse );
Resource gatherExplicitIndexFiles( Collection<StoreFileMetadata> files ) throws IOException { final Collection<ResourceIterator<File>> snapshots = new ArrayList<>(); for ( IndexImplementation indexProvider : explicitIndexProviders.allIndexProviders() ) { ResourceIterator<File> snapshot = indexProvider.listStoreFiles(); snapshots.add( snapshot ); getSnapshotFilesMetadata( snapshot, files ); } // Intentionally don't close the snapshot here, return it for closing by the consumer of // the targetFiles list. return new MultiResource( snapshots ); }
explicitIndexProvider.getProviderByName( existing.get( PROVIDER ) ), indexName, existing, config ); return config;
@Override public void flushAndForce( IOLimiter limiter ) { indexingService.forceAll( limiter ); labelScanStore.force( limiter ); for ( IndexImplementation index : explicitIndexProviderLookup.allIndexProviders() ) { index.force(); } neoStores.flush( limiter ); }
@Override public void migrate( DatabaseLayout directoryLayout, DatabaseLayout migrationLayout, ProgressReporter progressMonitor, String versionToMigrateFrom, String versionToMigrateTo ) throws IOException { IndexImplementation indexImplementation = explicitIndexProvider.getProviderByName( LUCENE_EXPLICIT_INDEX_PROVIDER_NAME ); if ( indexImplementation != null ) { RecordFormats from = RecordFormatSelector.selectForVersion( versionToMigrateFrom ); RecordFormats to = RecordFormatSelector.selectForVersion( versionToMigrateTo ); if ( !from.hasCompatibleCapabilities( to, CapabilityType.INDEX ) ) { originalExplicitIndexesRoot = indexImplementation.getIndexImplementationDirectory( directoryLayout ); migrationExplicitIndexesRoot = indexImplementation.getIndexImplementationDirectory( migrationLayout ); if ( isNotEmptyDirectory( originalExplicitIndexesRoot ) ) { migrateExplicitIndexes( progressMonitor ); explicitIndexMigrated = true; } } } else { log.debug( "Lucene index provider not found, nothing to migrate." ); } }
@Override public long getIndexStoreSize() { long size = 0L; // Add explicit indices for ( IndexImplementation index : explicitIndexProviderLookup.allIndexProviders() ) { size += FileUtils.size( fs, index.getIndexImplementationDirectory( databaseLayout ) ); } // Add schema index MutableLong schemaSize = new MutableLong(); indexProviderMap.accept( provider -> { File rootDirectory = provider.directoryStructure().rootDirectory(); if ( rootDirectory != null ) { schemaSize.add( FileUtils.size( fs, rootDirectory ) ); } // else this provider didn't have any persistent storage } ); size += schemaSize.longValue(); // Add label index size += FileUtils.size( fs, labelScanStore.getLabelScanStoreFile() ); return size; }
@Override public TransactionApplier newApplier( String providerName, boolean recovery ) { return provider.getProviderByName( providerName ).newApplier( recovery ); } }