currentLog = persistor.createLog(snapshot.getTimestamp()); persistor.writeSnapshot(snapshot); lastSnapshotTime = snapshotTime; long oldestRetainedTimestamp = persistor.deleteOldSnapshots(snapshotRetainCount); persistor.deleteLogsOlderThan(oldestRetainedTimestamp);
public synchronized void recoverState() { try { TransactionSnapshot lastSnapshot = persistor.getLatestSnapshot(); // if we failed before a snapshot could complete, we might not have one to restore if (lastSnapshot != null) { restoreSnapshot(lastSnapshot); } // replay any WALs since the last snapshot Collection<TransactionLog> logs = persistor.getLogsSince(lastSnapshotTime); if (logs != null) { replayLogs(logs); } } catch (IOException e) { LOG.error("Unable to read back transaction state:", e); throw Throwables.propagate(e); } }
private void refreshState() throws IOException { if (!initialized) { tryInit(); } // only continue if initialization was successful if (initialized) { long now = System.currentTimeMillis(); TransactionVisibilityState currentState = storage.getLatestTransactionVisibilityState(); if (currentState != null) { if (currentState.getTimestamp() < (now - 2 * snapshotRefreshFrequency)) { LOG.info("Current snapshot is old, will force a refresh on next run."); reset(); } else { latestState = currentState; LOG.info("Transaction state reloaded with snapshot from " + latestState.getTimestamp()); if (LOG.isDebugEnabled()) { LOG.debug("Latest transaction snapshot: " + latestState.toString()); } lastRefresh = now; } } else { LOG.info("No transaction state found."); } } }
try { storage = getStorage(conf); storage.startAndWait(); long now = System.currentTimeMillis(); long writePointer = 1; storage.writeSnapshot(snapshot); TransactionLog log = storage.createLog(now); log.append(dummyEdit); log.close(); storage.writeSnapshot(snapshot); log = storage.createLog(now + 1); log.append(dummyEdit); log.close(); storage.writeSnapshot(snapshot); log = storage.createLog(now + 2); log.append(dummyEdit); log.close(); storage.writeSnapshot(snapshot); log = storage.createLog(now + 3); log.append(dummyEdit); log.close(); storage.writeSnapshot(snapshot); log = storage.createLog(now + 4); log.append(dummyEdit);
txStorage.startAndWait(); TransactionSnapshot snapshot = txStorage.getLatestSnapshot(); TransactionVisibilityState txVisibilityState = txStorage.getLatestTransactionVisibilityState(); assertTransactionVisibilityStateEquals(snapshot, txVisibilityState); assertNotNull(snapshot); snapshot.getInProgress().entrySet().iterator().next(); assertNull(entry.getValue().getType()); txStorage.stopAndWait(); txStorage2.startAndWait(); TransactionSnapshot snapshot3 = txStorage2.getLatestSnapshot(); txStorage2.stopAndWait();
@Test public void testSnapshotPersistence() throws Exception { Configuration conf = getConfiguration("testSnapshotPersistence"); TransactionSnapshot snapshot = createRandomSnapshot(); TransactionStateStorage storage = getStorage(conf); try { storage.startAndWait(); storage.writeSnapshot(snapshot); TransactionSnapshot readSnapshot = storage.getLatestSnapshot(); assertNotNull(readSnapshot); assertEquals(snapshot, readSnapshot); } finally { storage.stopAndWait(); } }
try { storage = getStorage(conf); storage.startAndWait(); TransactionLog log = storage.createLog(time1); log.append(edit1); log.append(edit2); storage.stopAndWait();
try { long now = System.currentTimeMillis(); storage.startAndWait(); TransactionLog log = storage.createLog(now); for (TransactionEdit edit : edits) { log.append(edit); Collection<TransactionLog> logsToRead = storage.getLogsSince(now); storage.stopAndWait();
try { storage = getStorage(conf); storage.startAndWait(); storage.writeSnapshot(snapshot); TransactionLog log = storage.createLog(time2); log.append(edit1); log.append(edit2); storage.stopAndWait();
persistor.stopAndWait(); txMetricsCollector.stop(); timer.stop();
@Override public synchronized void doStart() { LOG.info("Starting transaction manager."); txMetricsCollector.start(); // start up the persistor persistor.startAndWait(); try { persistor.setupStorage(); } catch (IOException e) { Throwables.propagate(e); } // establish defaults in case there is no persistence clear(); // attempt to recover state from last run recoverState(); // start the periodic cleanup thread startCleanupThread(); startSnapshotThread(); startMetricsThread(); // initialize the WAL if we did not force a snapshot in recoverState() initLog(); // initialize next write pointer if needed if (lastWritePointer == 0) { lastWritePointer = getNextWritePointer(); readPointer = lastWritePointer; } notifyStarted(); }
private void initLog() { if (currentLog == null) { try { currentLog = persistor.createLog(System.currentTimeMillis()); } catch (IOException ioe) { throw Throwables.propagate(ioe); } } }
/** * Take a snapshot of the transaction state and serialize it into the given output stream. * @return whether a snapshot was taken. */ public boolean takeSnapshot(OutputStream out) throws IOException { TransactionSnapshot snapshot = getSnapshot(); if (snapshot != null) { persistor.writeSnapshot(out, snapshot); return true; } else { return false; } }
/** * Try to initialize the Configuration and TransactionStateStorage instances. Obtaining the Configuration may * fail until ReactorServiceMain has been started. */ private void tryInit() { try { Configuration conf = getSnapshotConfiguration(); if (conf != null) { // Since this is only used for background loading of transaction snapshots, we use the no-op metrics collector, // as there are no relevant metrics to report this.storage = new HDFSTransactionStateStorage(conf, new SnapshotCodecProvider(conf), new TxMetricsCollector()); this.storage.startAndWait(); this.snapshotRefreshFrequency = conf.getLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, TxConstants.Manager.DEFAULT_TX_SNAPSHOT_INTERVAL) * 1000; this.initialized = true; } else { LOG.info("Could not load configuration"); } } catch (Exception e) { LOG.info("Failed to initialize TransactionStateCache due to: " + e.getMessage()); } }
@Test public void testResetState() throws Exception { // have tx in progress, committing and committed then reset, // get the last snapshot and see that it is empty TransactionSystemClient client = getClient(); TransactionStateStorage stateStorage = getStateStorage(); Transaction tx1 = client.startShort(); Transaction tx2 = client.startShort(); client.canCommit(tx1, asList(C1, C2)); client.commit(tx1); client.canCommit(tx2, asList(C3, C4)); Transaction txPreReset = client.startShort(); long currentTs = System.currentTimeMillis(); client.resetState(); TransactionSnapshot snapshot = stateStorage.getLatestSnapshot(); Assert.assertTrue(snapshot.getTimestamp() >= currentTs); Assert.assertEquals(0, snapshot.getInvalid().size()); Assert.assertEquals(0, snapshot.getInProgress().size()); Assert.assertEquals(0, snapshot.getCommittingChangeSets().size()); Assert.assertEquals(0, snapshot.getCommittedChangeSets().size()); // confirm that transaction IDs are not reset Transaction txPostReset = client.startShort(); Assert.assertTrue("New tx ID should be greater than last ID before reset", txPostReset.getTransactionId() > txPreReset.getTransactionId()); }
private void startSnapshotThread() { if (snapshotFrequencyInSeconds > 0) { LOG.info("Starting periodic snapshot thread, frequency = " + snapshotFrequencyInSeconds + " seconds, location = " + persistor.getLocation()); this.snapshotThread = new DaemonThreadExecutor("tx-snapshot") { @Override
txStorage.startAndWait(); TransactionSnapshot snapshot = txStorage.getLatestSnapshot(); TransactionVisibilityState txVisibilityState = txStorage.getLatestTransactionVisibilityState(); assertTransactionVisibilityStateEquals(snapshot, txVisibilityState); inProgressTx.getCheckpointWritePointers().toLongArray()); txStorage.stopAndWait(); txStorage2.startAndWait(); snapshot = txStorage2.getLatestSnapshot(); Assert.assertTrue(snapshot.getInProgress().isEmpty()); txStorage2.stopAndWait();
try { storage = getStorage(conf); storage.startAndWait(); TransactionLog log = storage.createLog(time1); log.append(edit1); log.append(edit2); storage.stopAndWait();
persistor.stopAndWait(); txMetricsCollector.stop(); timer.stop();
@Override public synchronized void doStart() { LOG.info("Starting transaction manager."); txMetricsCollector.start(); // start up the persistor persistor.startAndWait(); try { persistor.setupStorage(); } catch (IOException e) { Throwables.propagate(e); } // establish defaults in case there is no persistence clear(); // attempt to recover state from last run recoverState(); // start the periodic cleanup thread startCleanupThread(); startSnapshotThread(); startMetricsThread(); // initialize the WAL if we did not force a snapshot in recoverState() initLog(); // initialize next write pointer if needed if (lastWritePointer == 0) { lastWritePointer = getNextWritePointer(); readPointer = lastWritePointer; } notifyStarted(); }