public static TTransaction wrap(Transaction tx) { return new TTransaction(tx.getTransactionId(), tx.getReadPointer(), Longs.asList(tx.getInvalids()), Longs.asList(tx.getInProgress()), tx.getFirstShortInProgress(), getTTransactionType(tx.getType()), tx.getWritePointer(), Longs.asList(tx.getCheckpointWritePointers()), getTVisibilityLevel(tx.getVisibilityLevel())); }
/** * Creates a new {@link Filter} for returning data only from visible transactions. * * @param tx the current transaction to apply. Only data visible to this transaction will be returned. * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false} * these will be interpreted as "delete" markers and the column will be filtered out * @param scanType the type of scan operation being performed * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead. */ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType, @Nullable Filter cellFilter) { this.tx = tx; this.oldestTsByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) { long familyTTL = ttlEntry.getValue(); oldestTsByFamily.put(ttlEntry.getKey(), familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS); } this.allowEmptyValues = allowEmptyValues; this.clearDeletes = scanType == ScanType.COMPACT_DROP_DELETES || (scanType == ScanType.USER_SCAN && tx.getVisibilityLevel() != Transaction.VisibilityLevel.SNAPSHOT_ALL); this.cellFilter = cellFilter; }
/** * Returns whether or not the given version should be visible to the current transaction. A version will be visible * if it was successfully committed prior to the current transaction starting, or was written by the current * transaction (using either the current write pointer or the write pointer from a prior checkpoint). * * @param version the data version to check for visibility * @return true if the version is visible, false if it should be hidden (filtered) * * @see #setVisibility(VisibilityLevel) to control whether the current write pointer is visible. */ public boolean isVisible(long version) { // either it was committed before or the change belongs to current tx return (version <= getReadPointer() && !isExcluded(version)) || (isCurrentWrite(version) && (visibilityLevel != VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT || writePointer != version)); }
/** * Creates a new transaction for a checkpoint operation, copying all members from the original transaction, * with the updated checkpoint write pointers. * * @param toCopy the original transaction containing the state to copy * @param writePointer the new write pointer to use for the transaction * @param checkpointPointers the list of write pointers added from checkpoints on the transaction */ public Transaction(Transaction toCopy, long writePointer, long[] checkpointPointers) { this(toCopy.getReadPointer(), toCopy.getTransactionId(), writePointer, toCopy.getInvalids(), toCopy.getInProgress(), toCopy.getFirstShortInProgress(), toCopy.getType(), checkpointPointers, toCopy.getVisibilityLevel()); }
} else if (tx.isVisible(kvTimestamp)) { if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL && tx.isCurrentWrite(kvTimestamp)) {
/** * Returns the maximum timestamp to use for time-range operations, based on the given transaction. * @param tx The current transaction * @return The maximum timestamp (exclusive) to use for time-range operations */ public static long getMaxVisibleTimestamp(Transaction tx) { // NOTE: +1 here because we want read up to writepointer inclusive, but timerange's end is exclusive // however, we also need to guard against overflow in the case write pointer is set to MAX_VALUE return tx.getWritePointer() < Long.MAX_VALUE ? tx.getWritePointer() + 1 : tx.getWritePointer(); }
@Override public Cell transformCell(Cell cell) throws IOException { // Convert Tephra deletes back into HBase deletes if (tx.getVisibilityLevel() == Transaction.VisibilityLevel.SNAPSHOT_ALL) { if (DeleteTracker.isFamilyDelete(cell)) { return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), null, cell.getTimestamp(), KeyValue.Type.DeleteFamily); } else if (isColumnDelete(cell)) { // Note: in some cases KeyValue.Type.Delete is used in Delete object, // and in some other cases KeyValue.Type.DeleteColumn is used. // Since Tephra cannot distinguish between the two, we return KeyValue.Type.DeleteColumn. // KeyValue.Type.DeleteColumn makes both CellUtil.isDelete and CellUtil.isDeleteColumns return true, and will // work in both cases. return new KeyValue(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell), cell.getTimestamp(), KeyValue.Type.DeleteColumn); } } return cell; }
/** * Returns the oldest visible timestamp for the given transaction, based on the TTLs configured for each column * family. If no TTL is set on any column family, the oldest visible timestamp will be {@code 0}. * @param ttlByFamily A map of column family name to TTL value (in milliseconds) * @param tx The current transaction * @return The oldest timestamp that will be visible for the given transaction and TTL configuration */ public static long getOldestVisibleTimestamp(Map<byte[], Long> ttlByFamily, Transaction tx) { long maxTTL = getMaxTTL(ttlByFamily); // we know that data will not be cleaned up while this tx is running up to this point as janitor uses it return maxTTL < Long.MAX_VALUE ? tx.getVisibilityUpperBound() - maxTTL * TxConstants.MAX_TX_PER_MS : 0; }
Transaction tx = client.startShort(); if (verbose) { LOG.info("Started tx details: " + tx.toString()); } else { LOG.info("Started tx: " + tx.getTransactionId() + ", readPointer: " + tx.getReadPointer() + ", invalids: " + tx.getInvalids().length + ", inProgress: " + tx.getInProgress().length);
private boolean hasConflicts(Transaction tx, Set<ChangeId> changeIds) { if (changeIds.isEmpty()) { return false; } for (Map.Entry<Long, Set<ChangeId>> changeSet : committedChangeSets.entrySet()) { // If commit time is greater than tx read-pointer, // basically not visible but committed means "tx committed after given tx was started" if (changeSet.getKey() > tx.getTransactionId()) { if (overlap(changeSet.getValue(), changeIds)) { return true; } } } return false; }
private Transaction startTx(long expiration, TransactionType type) { Transaction tx = null; long txid; // guard against changes to the transaction log while processing this.logReadLock.lock(); try { synchronized (this) { ensureAvailable(); txid = getNextWritePointer(); tx = createTransaction(txid, type); addInProgressAndAdvance(tx.getTransactionId(), tx.getVisibilityUpperBound(), expiration, type); } // appending to WAL out of global lock for concurrent performance // we should still be able to arrive at the same state even if log entries are out of order appendToLog(TransactionEdit.createStarted(tx.getTransactionId(), tx.getVisibilityUpperBound(), expiration, type)); } finally { this.logReadLock.unlock(); } return tx; }
if (inProgress.get(tx.getTransactionId()) == null) { if (invalid.contains(tx.getTransactionId())) { throw new TransactionNotInProgressException( String.format("canCommit() is called for transaction %d that is not in progress " + "(it is known to be invalid)", tx.getTransactionId())); } else { throw new TransactionNotInProgressException( String.format("canCommit() is called for transaction %d that is not in progress", tx.getTransactionId())); changeSet = committingChangeSets.remove(tx.getTransactionId()); doCommit(tx.getTransactionId(), tx.getWritePointer(), changeSet, commitPointer, addToCommitted); appendToLog(TransactionEdit.createCommitted(tx.getTransactionId(), changeSet, commitPointer, addToCommitted)); } finally { this.logReadLock.unlock();
Assert.assertTrue(tx4.getTransactionId() > tx3.getTransactionId()); Assert.assertTrue(tx2.isVisible(tx1.getTransactionId())); Assert.assertFalse(tx2.isVisible(tx3.getTransactionId())); Assert.assertFalse(tx2.isVisible(tx4.getTransactionId())); Assert.assertTrue(tx5.isVisible(tx1.getTransactionId())); Assert.assertTrue(tx5.isVisible(tx2.getTransactionId())); Assert.assertFalse(tx5.isVisible(tx3.getTransactionId())); Assert.assertFalse(tx5.isVisible(tx4.getTransactionId())); Assert.assertFalse(tx6.hasExcludes()); txManager.abort(tx6); Assert.assertTrue(txAfter.getTransactionId() > tx.getTransactionId()); } finally { if (storage != null) {
public void abort(Transaction tx) { // guard against changes to the transaction log while processing txMetricsCollector.rate("abort"); Stopwatch timer = new Stopwatch().start(); this.logReadLock.lock(); try { synchronized (this) { ensureAvailable(); doAbort(tx.getTransactionId(), tx.getCheckpointWritePointers(), tx.getType()); } appendToLog(TransactionEdit.createAborted(tx.getTransactionId(), tx.getType(), tx.getCheckpointWritePointers())); txMetricsCollector.histogram("abort.latency", (int) timer.elapsedMillis()); } finally { this.logReadLock.unlock(); } }
Transaction txOld = new Transaction(tx1.getReadPointer(), tx1.getTransactionId() - 1, new long[] {}, new long[] {}, Transaction.NO_TX_IN_PROGRESS, TransactionType.SHORT); Transaction txNew = new Transaction(tx1.getReadPointer(), tx1.getTransactionId() + 1, new long[] {}, new long[] {}, Transaction.NO_TX_IN_PROGRESS, TransactionType.SHORT);
Assert.assertFalse(txx.isVisible(tx1.getTransactionId())); Assert.assertFalse(txx.isVisible(tx2.getTransactionId())); Assert.assertFalse(txx.isVisible(tx3.getTransactionId())); Assert.assertEquals(tx3.getTransactionId(), (long) txm.getCurrentState().getInvalid().iterator().next()); Assert.assertEquals(1, txm.getExcludedListSize()); } finally {
@Test public void testTruncateInvalidTx() throws Exception { // Start few transactions and invalidate all of them TransactionSystemClient client = getClient(); Transaction tx1 = client.startLong(); Transaction tx2 = client.startShort(); Transaction tx3 = client.startLong(); client.invalidate(tx1.getTransactionId()); client.invalidate(tx2.getTransactionId()); client.invalidate(tx3.getTransactionId()); // Remove tx2 and tx3 from invalid list Assert.assertTrue(client.truncateInvalidTx(ImmutableSet.of(tx2.getTransactionId(), tx3.getTransactionId()))); Transaction tx = client.startShort(); // Only tx1 should be in invalid list now Assert.assertArrayEquals(new long[] {tx1.getTransactionId()}, tx.getInvalids()); client.abort(tx); }
@Override public boolean filterRowKey(byte[] buffer, int offset, int length) { // last 4 bytes in a row key counter = Bytes.toInt(buffer, offset + length - 4, Ints.BYTES); // row key is queue_name + writePointer + counter writePointer = Bytes.toLong(buffer, offset + queueNamePrefixLength, Longs.BYTES); // If writes later than the reader pointer, abort the loop, as entries that comes later are all uncommitted. // this is probably not needed due to the limit of the scan to the stop row, but to be safe... if (writePointer > transaction.getReadPointer()) { stopScan = true; return true; } // If the write is in the excluded list, ignore it. if (transaction.isExcluded(writePointer)) { return true; } return false; }
private void assertVisibility(Set<Long> priorCommitted, Set<Long> postCommitted, Set<Long> priorInvalids, Set<Long> postInvalids, Set<Long> priorInProgress, Set<Long> postInProgress, Set<Long> visibleCurrent, Set<Long> notVisibleCurrent, Transaction tx) { // Verify visible snapshots of tx are visible for (long t : visibleCurrent) { Assert.assertTrue("Assertion error for version = " + t, tx.isVisible(t)); } // Verify not visible snapshots of tx are not visible for (long t : notVisibleCurrent) { Assert.assertFalse("Assertion error for version = " + t, tx.isVisible(t)); } // Verify prior committed versions are visible for (long t : priorCommitted) { Assert.assertTrue("Assertion error for version = " + t, tx.isVisible(t)); } // Verify versions committed after tx started, and not part of tx are not visible for (long t : postCommitted) { Assert.assertFalse("Assertion error for version = " + t, tx.isVisible(t)); } // Verify invalid and in-progress versions are not visible for (long t : Iterables.concat(priorInvalids, postInvalids, priorInProgress, postInProgress)) { Assert.assertFalse("Assertion error for version = " + t, tx.isVisible(t)); } }
long txId = originalTx.getTransactionId(); long newWritePointer = 0; checkpointedTx = new Transaction(originalTx, newWritePointer, parentTx.getCheckpointWritePointers().toLongArray());