/** * Build log files that can access and operate only on active set of log files without ability to * rotate and create any new one. Appending to current log file still possible. * Store and external components access available in read only mode. * @param databaseLayout store directory * @param fileSystem log file system * @param pageCache page cache for read only store info access */ public static LogFilesBuilder activeFilesBuilder( DatabaseLayout databaseLayout, FileSystemAbstraction fileSystem, PageCache pageCache ) { LogFilesBuilder builder = builder( databaseLayout, fileSystem ); builder.pageCache = pageCache; builder.readOnly = true; return builder; }
private LogFiles createLogFiles( TransactionIdStore transactionIdStore, FileSystemAbstraction fileSystemAbstraction ) throws IOException { SimpleLogVersionRepository logVersionRepository = new SimpleLogVersionRepository(); return LogFilesBuilder.builder( databaseLayout, fileSystemAbstraction ) .withTransactionIdStore(transactionIdStore) .withLogVersionRepository( logVersionRepository ) .build(); }
@Test public void shouldSuppressFailureToCloseChannelInFailedAttemptToReadHeaderAfterOpen() throws Exception { // GIVEN a file which returns 1/2 log header size worth of bytes FileSystemAbstraction fs = mock( FileSystemAbstraction.class ); LogFiles logFiles = LogFilesBuilder.builder( directory.databaseLayout(), fs ) .withTransactionIdStore( transactionIdStore ) .withLogVersionRepository( logVersionRepository ).build(); int logVersion = 0; File logFile = logFiles.getLogFileForVersion( logVersion ); StoreChannel channel = mock( StoreChannel.class ); when( channel.read( any( ByteBuffer.class ) ) ).thenReturn( LogHeader.LOG_HEADER_SIZE / 2 ); when( fs.fileExists( logFile ) ).thenReturn( true ); when( fs.open( eq( logFile ), any( OpenMode.class ) ) ).thenReturn( channel ); doThrow( IOException.class ).when( channel ).close(); // WHEN try { logFiles.openForVersion( logVersion ); fail( "Should have failed" ); } catch ( IncompleteLogHeaderException e ) { // THEN good verify( channel ).close(); assertEquals( 1, e.getSuppressed().length ); assertTrue( e.getSuppressed()[0] instanceof IOException ); } }
@Test( expected = NullPointerException.class ) public void failToBuildFullContextWithoutTransactionIdStore() throws IOException { builder( testDirectory.databaseLayout(), fileSystem ).withLogVersionRepository( new SimpleLogVersionRepository( 2 ) ).buildContext(); }
private LogFiles getLogFiles( SimpleLogVersionRepository logVersionRepository, SimpleTransactionIdStore transactionIdStore ) throws IOException { return LogFilesBuilder.builder( testDirectory.databaseLayout(), fileSystem.get() ) .withLogVersionRepository( logVersionRepository ).withTransactionIdStore( transactionIdStore ).build(); }
@Test( expected = NullPointerException.class ) public void failToBuildFullContextWithoutLogVersionRepo() throws IOException { builder( testDirectory.databaseLayout(), fileSystem ).withTransactionIdStore( new SimpleTransactionIdStore() ).buildContext(); }
private LogFiles buildDefaultLogFiles() throws IOException { return LogFilesBuilder.builder( directory.databaseLayout(), fileSystemRule ) .withLogVersionRepository( new SimpleLogVersionRepository() ) .withTransactionIdStore( new SimpleTransactionIdStore() ).build(); }
private LogFiles createLogFiles() throws IOException { return LogFilesBuilder .builder( testDirectory.databaseLayout(), fileSystemRule ) .withLogFileName( filename ) .withTransactionIdStore( new SimpleTransactionIdStore() ) .withLogVersionRepository( new SimpleLogVersionRepository() ) .build(); }
@Before public void setUp() throws Exception { storeDir = directory.storeDir(); logFiles = LogFilesBuilder.builder( directory.databaseLayout(), fileSystemRule.get() ) .withLogVersionRepository( logVersionRepository ) .withTransactionIdStore( transactionIdStore ) .build(); }
@Test public void buildContextWithCustomAbsoluteLogFilesLocations() throws Throwable { File customLogDirectory = testDirectory.directory( "absoluteCustomLogDirectory" ); Config customLogLocationConfig = Config.defaults( logical_logs_location, customLogDirectory.getAbsolutePath() ); LogFiles logFiles = builder( testDirectory.databaseLayout(), fileSystem ).withConfig( customLogLocationConfig ) .withLogVersionRepository( new SimpleLogVersionRepository() ) .withTransactionIdStore( new SimpleTransactionIdStore() ).build(); logFiles.init(); logFiles.start(); assertEquals( customLogDirectory, logFiles.getHighestLogFile().getParentFile() ); logFiles.shutdown(); }
@Before public void setUp() throws IOException { LogVersionRepository logVersionRepository = new SimpleLogVersionRepository(); SimpleTransactionIdStore transactionIdStore = new SimpleTransactionIdStore(); LogFiles logFiles = LogFilesBuilder.builder( directory.databaseLayout(), fs ) .withLogVersionRepository( logVersionRepository ) .withTransactionIdStore( transactionIdStore ) .build(); life.add( logFiles ); logFile = logFiles.getLogFile(); }
@Test public void buildContextWithCustomLogFilesLocations() throws Throwable { String customLogLocation = "customLogLocation"; Config customLogLocationConfig = Config.defaults( logical_logs_location, customLogLocation ); DatabaseLayout databaseLayout = testDirectory.databaseLayout(); LogFiles logFiles = builder( databaseLayout, fileSystem ).withConfig( customLogLocationConfig ) .withLogVersionRepository( new SimpleLogVersionRepository() ) .withTransactionIdStore( new SimpleTransactionIdStore() ).build(); logFiles.init(); logFiles.start(); assertEquals( databaseLayout.file( customLogLocation ), logFiles.getHighestLogFile().getParentFile() ); logFiles.shutdown(); }
private void addCorruptedCommandsToLastLogFile() throws IOException { PositiveLogFilesBasedLogVersionRepository versionRepository = new PositiveLogFilesBasedLogVersionRepository( logFiles ); LogFiles internalLogFiles = LogFilesBuilder.builder( directory.databaseLayout(), fileSystemRule ) .withLogVersionRepository( versionRepository ) .withTransactionIdStore( new SimpleTransactionIdStore() ).build(); try ( Lifespan lifespan = new Lifespan( internalLogFiles ) ) { LogFile transactionLogFile = internalLogFiles.getLogFile(); FlushablePositionAwareChannel channel = transactionLogFile.getWriter(); TransactionLogWriter writer = new TransactionLogWriter( new CorruptedLogEntryWriter( channel ) ); Collection<StorageCommand> commands = new ArrayList<>(); commands.add( new Command.PropertyCommand( new PropertyRecord( 1 ), new PropertyRecord( 2 ) ) ); commands.add( new Command.NodeCommand( new NodeRecord( 2 ), new NodeRecord( 3 ) ) ); PhysicalTransactionRepresentation transaction = new PhysicalTransactionRepresentation( commands ); writer.append( transaction, 1000 ); } }
@Test public void shouldOpenCleanStore() throws Exception { // GIVEN TransactionIdStore transactionIdStore = new SimpleTransactionIdStore(); TransactionMetadataCache positionCache = new TransactionMetadataCache(); LifeSupport life = new LifeSupport(); final LogFiles logFiles = LogFilesBuilder.builder( dir.databaseLayout(), fileSystemRule.get() ) .withTransactionIdStore( transactionIdStore ) .withLogVersionRepository( mock( LogVersionRepository.class ) ).build(); life.add( logFiles ); life.add( new BatchingTransactionAppender( logFiles, NO_ROTATION, positionCache, transactionIdStore, BYPASS, DATABASE_HEALTH ) ); try { // WHEN life.start(); } finally { life.shutdown(); } }
@Test public void pruningStrategyShouldBeDynamic() throws IOException { CheckPointer checkPointer = getInstanceFromDb( CheckPointer.class ); Config config = getInstanceFromDb( Config.class ); FileSystemAbstraction fs = getInstanceFromDb( FileSystemAbstraction.class ); LogFiles logFiles = LogFilesBuilder.builder( db.databaseLayout(), fs ) .withLogVersionRepository( new SimpleLogVersionRepository() ) .withLastCommittedTransactionIdSupplier( () -> 1 ) .withTransactionIdStore( new SimpleTransactionIdStore() ).build(); // Force transaction log rotation writeTransactionsAndRotateTwice(); // Checkpoint to make sure strategy is evaluated checkPointer.forceCheckPoint( triggerInfo ); // Make sure file is still there since we have disable pruning assertThat( countTransactionLogs( logFiles ), is( 3 ) ); // Change pruning to true config.updateDynamicSetting( keep_logical_logs.name(), "false", "test" ); // Checkpoint to make sure strategy is evaluated checkPointer.forceCheckPoint( triggerInfo ); // Make sure file is removed assertThat( countTransactionLogs( logFiles ), is( 2 ) ); }
@Test public void buildDefaultContext() throws IOException { TransactionLogFilesContext context = builder( testDirectory.databaseLayout(), fileSystem ) .withLogVersionRepository( new SimpleLogVersionRepository( 2 ) ) .withTransactionIdStore( new SimpleTransactionIdStore() ).buildContext(); assertEquals( fileSystem, context.getFileSystem() ); assertNotNull( context.getLogEntryReader() ); assertSame( LogFileCreationMonitor.NO_MONITOR, context.getLogFileCreationMonitor() ); assertEquals( ByteUnit.mebiBytes( 250 ), context.getRotationThreshold().get() ); assertEquals( 1, context.getLastCommittedTransactionId() ); assertEquals( 2, context.getLogVersionRepository().getCurrentLogVersion() ); }
@Test public void buildDefaultContextWithDependencies() throws IOException { SimpleLogVersionRepository logVersionRepository = new SimpleLogVersionRepository( 2 ); SimpleTransactionIdStore transactionIdStore = new SimpleTransactionIdStore(); Dependencies dependencies = new Dependencies(); dependencies.satisfyDependency( logVersionRepository ); dependencies.satisfyDependency( transactionIdStore ); TransactionLogFilesContext context = builder( testDirectory.databaseLayout(), fileSystem ).withDependencies( dependencies ).buildContext(); assertEquals( fileSystem, context.getFileSystem() ); assertNotNull( context.getLogEntryReader() ); assertSame( LogFileCreationMonitor.NO_MONITOR, context.getLogFileCreationMonitor() ); assertEquals( ByteUnit.mebiBytes( 250 ), context.getRotationThreshold().get() ); assertEquals( 1, context.getLastCommittedTransactionId() ); assertEquals( 2, context.getLogVersionRepository().getCurrentLogVersion() ); }
@Test public void shouldOpenInFreshDirectoryAndFinallyAddHeader() throws Exception { // GIVEN String name = "log"; FileSystemAbstraction fs = fileSystemRule.get(); LogFiles logFiles = LogFilesBuilder.builder( directory.databaseLayout(), fs ) .withTransactionIdStore( transactionIdStore ) .withLogVersionRepository( logVersionRepository ).build(); // WHEN life.start(); life.add( logFiles ); life.shutdown(); // THEN File file = LogFilesBuilder.logFilesBasedOnlyBuilder( directory.databaseDir(), fs ).build().getLogFileForVersion( 1L ); LogHeader header = readLogHeader( fs, file ); assertEquals( 1L, header.logVersion ); assertEquals( 2L, header.lastCommittedTxId ); }
@Test public void shouldWriteSomeDataIntoTheLog() throws Exception { // GIVEN String name = "log"; FileSystemAbstraction fs = fileSystemRule.get(); LogFiles logFiles = LogFilesBuilder.builder( directory.databaseLayout(), fs ) .withTransactionIdStore( transactionIdStore ) .withLogVersionRepository( logVersionRepository ).build(); life.start(); life.add( logFiles ); // WHEN FlushablePositionAwareChannel writer = logFiles.getLogFile().getWriter(); LogPositionMarker positionMarker = new LogPositionMarker(); writer.getCurrentPosition( positionMarker ); int intValue = 45; long longValue = 4854587; writer.putInt( intValue ); writer.putLong( longValue ); writer.prepareForFlush().flush(); // THEN try ( ReadableClosableChannel reader = logFiles.getLogFile().getReader( positionMarker.newPosition() ) ) { assertEquals( intValue, reader.getInt() ); assertEquals( longValue, reader.getLong() ); } }
@Test public void skipLogFileWithoutHeader() throws IOException { FileSystemAbstraction fs = fileSystemRule.get(); LogFiles logFiles = LogFilesBuilder.builder( directory.databaseLayout(), fs ) .withTransactionIdStore( transactionIdStore ) .withLogVersionRepository( logVersionRepository ).build(); life.add( logFiles ); life.start(); // simulate new file without header presence logVersionRepository.incrementAndGetVersion(); fs.create( logFiles.getLogFileForVersion( logVersionRepository.getCurrentLogVersion() ) ).close(); transactionIdStore.transactionCommitted( 5L, 5L, 5L ); PhysicalLogicalTransactionStore.LogVersionLocator versionLocator = new PhysicalLogicalTransactionStore.LogVersionLocator( 4L ); logFiles.accept( versionLocator ); LogPosition logPosition = versionLocator.getLogPosition(); assertEquals( 1, logPosition.getLogVersion() ); }