private StoreMigrator newStoreMigrator() { return new StoreMigrator( fileSystemRule, pageCache, Config.defaults(), NullLogService.getInstance(), jobScheduler ); }
@Override public void migrate( DatabaseLayout directoryLayout, DatabaseLayout migrationLayout, ProgressReporter progressReporter, String versionToMigrateFrom, String versionToMigrateTo ) throws IOException { super.migrate( directoryLayout, migrationLayout, progressReporter, versionToMigrateFrom, versionToMigrateTo ); throw new RuntimeException( "This upgrade is failing" ); } };
@Test public void writeAndReadLastTxInformation() throws IOException { StoreMigrator migrator = newStoreMigrator(); TransactionId writtenTxId = new TransactionId( random.nextLong(), random.nextLong(), random.nextLong() ); migrator.writeLastTxInformation( directory.databaseLayout(), writtenTxId ); TransactionId readTxId = migrator.readLastTxInformation( directory.databaseLayout() ); assertEquals( writtenTxId, readTxId ); }
TransactionId lastTxInfo = extractTransactionIdInformation( neoStore, lastTxId ); LogPosition lastTxLogPosition = extractTransactionLogPosition( neoStore, directoryLayout, lastTxId ); writeLastTxInformation( migrationLayout, lastTxInfo ); writeLastTxLogPosition( migrationLayout, lastTxLogPosition ); RecordFormats newFormat = selectForVersion( versionToMigrateTo ); if ( FormatFamily.isHigherFamilyFormat( newFormat, oldFormat ) || (FormatFamily.isSameFamily( oldFormat, newFormat ) && isDifferentCapabilities( oldFormat, newFormat )) ) migrateWithBatchImporter( directoryLayout, migrationLayout, lastTxId, lastTxInfo.checksum(), lastTxLogPosition.getLogVersion(), lastTxLogPosition.getByteOffset(), progressReporter, oldFormat, newFormat ); LogPosition logPosition = readLastTxLogPosition( migrationLayout ); updateOrAddNeoStoreFieldsAsPartOfMigration( migrationLayout, directoryLayout, versionToMigrateTo, logPosition );
@Test public void writeAndReadLastTxLogPosition() throws IOException { StoreMigrator migrator = newStoreMigrator(); LogPosition writtenLogPosition = new LogPosition( random.nextLong(), random.nextLong() ); migrator.writeLastTxLogPosition( directory.databaseLayout(), writtenLogPosition ); LogPosition readLogPosition = migrator.readLastTxLogPosition( directory.databaseLayout() ); assertEquals( writtenLogPosition, readLogPosition ); }
StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, jobScheduler ); CountsMigrator countsMigrator = new CountsMigrator( fs, pageCache, CONFIG ); DatabaseLayout migrationLayout = directory.databaseLayout( StoreUpgrader.MIGRATION_DIRECTORY ); migrator.migrate( databaseLayout, migrationLayout, progressMonitor.startSection( "section" ), versionToMigrateFrom, upgradableDatabase.currentVersion() ); countsMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, jobScheduler ); countsMigrator = new CountsMigrator( fs, pageCache, CONFIG ); migrator.moveMigratedFiles( migrationLayout, databaseLayout, versionToMigrateFrom, upgradableDatabase.currentVersion() ); countsMigrator.moveMigratedFiles( migrationLayout, databaseLayout, versionToMigrateFrom,
@Test public void shouldExtractTransactionInformationFromMetaDataStore() throws Exception { // given // ... variables long txId = 42; long checksum = 123456789123456789L; long timestamp = 919191919191919191L; TransactionId expected = new TransactionId( txId, checksum, timestamp ); // ... and files DatabaseLayout databaseLayout = directory.databaseLayout(); File neoStore = databaseLayout.metadataStore(); neoStore.createNewFile(); // ... and mocks Config config = mock( Config.class ); LogService logService = mock( LogService.class ); // when // ... data in record setRecord( pageCache, neoStore, LAST_TRANSACTION_ID, txId ); setRecord( pageCache, neoStore, LAST_TRANSACTION_CHECKSUM, checksum ); setRecord( pageCache, neoStore, LAST_TRANSACTION_COMMIT_TIMESTAMP, timestamp ); // ... and with migrator StoreMigrator migrator = new StoreMigrator( fileSystemRule.get(), pageCache, config, logService, jobScheduler ); TransactionId actual = migrator.extractTransactionIdInformation( neoStore, txId ); // then assertEquals( expected, actual ); }
@Test public void shouldComputeTheLastTxInfoCorrectly() throws Exception { // given DatabaseLayout databaseLayout = directory.databaseLayout(); File prepare = directory.directory( "prepare" ); MigrationTestUtils.prepareSampleLegacyDatabase( version, fs, databaseLayout.databaseDirectory(), prepare ); // and a state of the migration saying that it has done the actual migration LogService logService = NullLogService.getInstance(); PageCache pageCache = pageCacheRule.getPageCache( fs ); LogTailScanner tailScanner = getTailScanner( databaseLayout.databaseDirectory() ); UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache, tailScanner ); String versionToMigrateFrom = upgradableDatabase.checkUpgradable( databaseLayout ).storeVersion(); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, jobScheduler ); DatabaseLayout migrationLayout = directory.databaseLayout( StoreUpgrader.MIGRATION_DIRECTORY ); // when migrator.migrate( databaseLayout, migrationLayout, progressMonitor.startSection( "section" ), versionToMigrateFrom, upgradableDatabase.currentVersion() ); // then assertTrue( txIdComparator.apply( migrator.readLastTxInformation( migrationLayout ) ) ); }
@Test public void shouldComputeTheLastTxLogPositionCorrectly() throws Throwable { // GIVEN a legacy database DatabaseLayout databaseLayout = directory.databaseLayout(); File prepare = directory.directory( "prepare" ); MigrationTestUtils.prepareSampleLegacyDatabase( version, fs, databaseLayout.databaseDirectory(), prepare ); // and a state of the migration saying that it has done the actual migration LogService logService = NullLogService.getInstance(); PageCache pageCache = pageCacheRule.getPageCache( fs ); LogTailScanner tailScanner = getTailScanner( databaseLayout.databaseDirectory() ); UpgradableDatabase upgradableDatabase = getUpgradableDatabase( pageCache, tailScanner ); String versionToMigrateFrom = upgradableDatabase.checkUpgradable( databaseLayout ).storeVersion(); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, jobScheduler ); DatabaseLayout migrationLayout = directory.databaseLayout( StoreUpgrader.MIGRATION_DIRECTORY ); // WHEN migrating migrator.migrate( databaseLayout, migrationLayout, progressMonitor.startSection( "section" ), versionToMigrateFrom, upgradableDatabase.currentVersion() ); // THEN it should compute the correct last tx log position assertEquals( expectedLogPosition, migrator.readLastTxLogPosition( migrationLayout ) ); }
private void extractTransactionalInformationFromLogs( String path, File customLogLocation, DatabaseLayout databaseLayout, File storeDir ) throws IOException { LogService logService = new SimpleLogService( NullLogProvider.getInstance(), NullLogProvider.getInstance() ); File neoStore = databaseLayout.metadataStore(); GraphDatabaseService database = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( storeDir ) .setConfig( logical_logs_location, path ).newGraphDatabase(); for ( int i = 0; i < 10; i++ ) { try ( Transaction transaction = database.beginTx() ) { Node node = database.createNode(); transaction.success(); } } database.shutdown(); MetaDataStore.setRecord( pageCache, neoStore, MetaDataStore.Position.LAST_CLOSED_TRANSACTION_LOG_VERSION, MetaDataRecordFormat.FIELD_NOT_PRESENT ); Config config = Config.defaults( logical_logs_location, path ); StoreMigrator migrator = new StoreMigrator( fileSystemRule.get(), pageCache, config, logService, jobScheduler ); LogPosition logPosition = migrator.extractTransactionLogPosition( neoStore, databaseLayout, 100 ); File[] logFiles = customLogLocation.listFiles(); assertNotNull( logFiles ); assertEquals( 0, logPosition.getLogVersion() ); assertEquals( logFiles[0].length(), logPosition.getByteOffset() ); }
TransactionId lastTxInfo = readLastTxInformation( migrationStructure );
@Override public void moveMigratedFiles( DatabaseLayout migrationLayout, DatabaseLayout directoryLayout, String versionToUpgradeFrom, String versionToMigrateTo ) throws IOException { super.moveMigratedFiles( migrationLayout, directoryLayout, versionToUpgradeFrom, versionToMigrateTo ); throw new RuntimeException( "This upgrade is failing" ); } };
TransactionId lastTxInfo = extractTransactionIdInformation( neoStore, lastTxId ); LogPosition lastTxLogPosition = extractTransactionLogPosition( neoStore, directoryLayout, lastTxId ); writeLastTxInformation( migrationLayout, lastTxInfo ); writeLastTxLogPosition( migrationLayout, lastTxLogPosition ); RecordFormats newFormat = selectForVersion( versionToMigrateTo ); if ( FormatFamily.isHigherFamilyFormat( newFormat, oldFormat ) || (FormatFamily.isSameFamily( oldFormat, newFormat ) && isDifferentCapabilities( oldFormat, newFormat )) ) migrateWithBatchImporter( directoryLayout, migrationLayout, lastTxId, lastTxInfo.checksum(), lastTxLogPosition.getLogVersion(), lastTxLogPosition.getByteOffset(), progressReporter, oldFormat, newFormat ); LogPosition logPosition = readLastTxLogPosition( migrationLayout ); updateOrAddNeoStoreFieldsAsPartOfMigration( migrationLayout, directoryLayout, versionToMigrateTo, logPosition );
StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, jobScheduler ); DatabaseLayout migrationLayout = directory.databaseLayout( StoreUpgrader.MIGRATION_DIRECTORY ); migrator.migrate( databaseLayout, migrationLayout, progressMonitor.startSection( "section" ), versionToMigrateFrom, upgradableDatabase.currentVersion() ); countsMigrator.migrate( databaseLayout, migrationLayout, progressMonitor.startSection( "section" ), versionToMigrateFrom, upgradableDatabase.currentVersion() ); migrator.moveMigratedFiles( migrationLayout, databaseLayout, versionToMigrateFrom, upgradableDatabase.currentVersion() ); countsMigrator.moveMigratedFiles( migrationLayout, databaseLayout, versionToMigrateFrom,
@Test public void shouldGenerateTransactionInformationWhenLogsAreEmpty() throws Exception { // given long txId = 1; DatabaseLayout databaseLayout = directory.databaseLayout(); File neoStore = databaseLayout.metadataStore(); neoStore.createNewFile(); Config config = mock( Config.class ); LogService logService = new SimpleLogService( NullLogProvider.getInstance(), NullLogProvider.getInstance() ); // when // ... transaction info not in neo store assertEquals( FIELD_NOT_PRESENT, getRecord( pageCache, neoStore, LAST_TRANSACTION_ID ) ); assertEquals( FIELD_NOT_PRESENT, getRecord( pageCache, neoStore, LAST_TRANSACTION_CHECKSUM ) ); assertEquals( FIELD_NOT_PRESENT, getRecord( pageCache, neoStore, LAST_TRANSACTION_COMMIT_TIMESTAMP ) ); // ... and with migrator StoreMigrator migrator = new StoreMigrator( fileSystemRule.get(), pageCache, config, logService, jobScheduler ); TransactionId actual = migrator.extractTransactionIdInformation( neoStore, txId ); // then assertEquals( txId, actual.transactionId() ); assertEquals( TransactionIdStore.BASE_TX_CHECKSUM, actual.checksum() ); assertEquals( TransactionIdStore.BASE_TX_COMMIT_TIMESTAMP, actual.commitTimestamp() ); }
TransactionId lastTxInfo = readLastTxInformation( migrationStructure );
StoreMigrator migrator = new StoreMigrator( fs, pageCache, CONFIG, logService, jobScheduler ); CountsMigrator countsMigrator = new CountsMigrator( fs, pageCache, CONFIG ); DatabaseLayout migrationLayout = directory.databaseLayout( StoreUpgrader.MIGRATION_DIRECTORY ); migrator.migrate( databaseLayout, migrationLayout, progressMonitor.startSection( "section" ), versionToMigrateFrom, upgradableDatabase.currentVersion() ); countsMigrator .migrate( databaseLayout, migrationLayout, progressMonitor.startSection( "section" ), versionToMigrateFrom, upgradableDatabase.currentVersion() ); migrator.moveMigratedFiles( migrationLayout, databaseLayout, versionToMigrateFrom, upgradableDatabase.currentVersion() ); countsMigrator.moveMigratedFiles( migrationLayout, databaseLayout, versionToMigrateFrom,
/** * Performs construction of {@link StoreUpgrader} and all of the necessary participants and performs store * migration if that is required. * @param directoryStructure database to migrate */ public void migrate( DatabaseLayout directoryStructure ) { LogProvider logProvider = logService.getInternalLogProvider(); UpgradableDatabase upgradableDatabase = new UpgradableDatabase( new StoreVersionCheck( pageCache ), format, tailScanner ); StoreUpgrader storeUpgrader = new StoreUpgrader( upgradableDatabase, progressMonitor, config, fs, pageCache, logProvider ); ExplicitIndexMigrator explicitIndexMigrator = new ExplicitIndexMigrator( fs, explicitIndexProvider, logProvider ); StoreMigrator storeMigrator = new StoreMigrator( fs, pageCache, config, logService, jobScheduler ); NativeLabelScanStoreMigrator nativeLabelScanStoreMigrator = new NativeLabelScanStoreMigrator( fs, pageCache, config ); CountsMigrator countsMigrator = new CountsMigrator( fs, pageCache, config ); indexProviderMap.accept( provider -> storeUpgrader.addParticipant( provider.storeMigrationParticipant( fs, pageCache ) ) ); storeUpgrader.addParticipant( explicitIndexMigrator ); storeUpgrader.addParticipant( storeMigrator ); storeUpgrader.addParticipant( nativeLabelScanStoreMigrator ); storeUpgrader.addParticipant( countsMigrator ); storeUpgrader.migrateIfNeeded( directoryStructure ); } }
@Test public void shouldGenerateTransactionInformationWhenLogsNotPresent() throws Exception { // given long txId = 42; DatabaseLayout databaseLayout = directory.databaseLayout(); File neoStore = databaseLayout.metadataStore(); neoStore.createNewFile(); Config config = mock( Config.class ); LogService logService = new SimpleLogService( NullLogProvider.getInstance(), NullLogProvider.getInstance() ); // when // ... transaction info not in neo store assertEquals( FIELD_NOT_PRESENT, getRecord( pageCache, neoStore, LAST_TRANSACTION_ID ) ); assertEquals( FIELD_NOT_PRESENT, getRecord( pageCache, neoStore, LAST_TRANSACTION_CHECKSUM ) ); assertEquals( FIELD_NOT_PRESENT, getRecord( pageCache, neoStore, LAST_TRANSACTION_COMMIT_TIMESTAMP ) ); // ... and with migrator StoreMigrator migrator = new StoreMigrator( fileSystemRule.get(), pageCache, config, logService, jobScheduler ); TransactionId actual = migrator.extractTransactionIdInformation( neoStore, txId ); // then assertEquals( txId, actual.transactionId() ); assertEquals( TransactionIdStore.UNKNOWN_TX_CHECKSUM, actual.checksum() ); assertEquals( TransactionIdStore.UNKNOWN_TX_COMMIT_TIMESTAMP, actual.commitTimestamp() ); }
@Test public void shouldNotMigrateFilesForVersionsWithSameCapability() throws Exception { // Prepare migrator and file StoreMigrator migrator = newStoreMigrator(); DatabaseLayout dbLayout = directory.databaseLayout(); File neoStore = dbLayout.metadataStore(); neoStore.createNewFile(); // Monitor what happens MyProgressReporter progressReporter = new MyProgressReporter(); // Migrate with two storeversions that have the same FORMAT capabilities migrator.migrate( dbLayout, directory.databaseLayout( "migrationDir" ), progressReporter, StandardV3_0.STORE_VERSION, StandardV3_2.STORE_VERSION ); // Should not have started any migration assertFalse( progressReporter.started ); }