/** * Returns the oldest visible timestamp for the given transaction, based on the TTLs configured for each column * family. If no TTL is set on any column family, the oldest visible timestamp will be {@code 0}. * @param ttlByFamily A map of column family name to TTL value (in milliseconds) * @param tx The current transaction * @return The oldest timestamp that will be visible for the given transaction and TTL configuration */ public static long getOldestVisibleTimestamp(Map<byte[], Long> ttlByFamily, Transaction tx) { long maxTTL = getMaxTTL(ttlByFamily); // we know that data will not be cleaned up while this tx is running up to this point as janitor uses it return maxTTL < Long.MAX_VALUE ? tx.getVisibilityUpperBound() - maxTTL * TxConstants.MAX_TX_PER_MS : 0; }
/** * Returns the oldest visible timestamp for the given transaction, based on the TTLs configured for each column * family. If no TTL is set on any column family, the oldest visible timestamp will be {@code 0}. * @param ttlByFamily A map of column family name to TTL value (in milliseconds) * @param tx The current transaction * @return The oldest timestamp that will be visible for the given transaction and TTL configuration */ public static long getOldestVisibleTimestamp(Map<byte[], Long> ttlByFamily, Transaction tx) { long maxTTL = getMaxTTL(ttlByFamily); // we know that data will not be cleaned up while this tx is running up to this point as janitor uses it return maxTTL < Long.MAX_VALUE ? tx.getVisibilityUpperBound() - maxTTL * TxConstants.MAX_TX_PER_MS : 0; }
/** * Creates a new {@link Filter} for returning data only from visible transactions. * * @param tx the current transaction to apply. Only data visible to this transaction will be returned. * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false} * these will be interpreted as "delete" markers and the column will be filtered out * @param scanType the type of scan operation being performed * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead. */ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType, @Nullable Filter cellFilter) { this.tx = tx; this.oldestTsByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) { long familyTTL = ttlEntry.getValue(); oldestTsByFamily.put(ttlEntry.getKey(), familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS); } this.allowEmptyValues = allowEmptyValues; this.clearDeletes = scanType == ScanType.COMPACT_DROP_DELETES || (scanType == ScanType.USER_SCAN && tx.getVisibilityLevel() != Transaction.VisibilityLevel.SNAPSHOT_ALL); this.cellFilter = cellFilter; }
/** * Creates a new {@link Filter} for returning data only from visible transactions. * * @param tx the current transaction to apply. Only data visible to this transaction will be returned. * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false} * these will be interpreted as "delete" markers and the column will be filtered out * @param scanType the type of scan operation being performed * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead. */ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType, @Nullable Filter cellFilter) { this.tx = tx; this.oldestTsByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) { long familyTTL = ttlEntry.getValue(); oldestTsByFamily.put(ttlEntry.getKey(), familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS); } this.allowEmptyValues = allowEmptyValues; this.clearDeletes = scanType == ScanType.COMPACT_DROP_DELETES || (scanType == ScanType.USER_SCAN && tx.getVisibilityLevel() != Transaction.VisibilityLevel.SNAPSHOT_ALL); this.cellFilter = cellFilter; }
/** * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions. * * @param tx the current transaction to apply. Only data visible to this transaction will be returned. * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false} * these will be interpreted as "delete" markers and the column will be filtered out * @param scanType the type of scan operation being performed * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead. */ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType, @Nullable Filter cellFilter) { this.tx = tx; this.oldestTsByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) { long familyTTL = ttlEntry.getValue(); oldestTsByFamily.put(ttlEntry.getKey(), familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS); } this.allowEmptyValues = allowEmptyValues; this.clearDeletes = scanType == ScanType.COMPACT_DROP_DELETES || (scanType == ScanType.USER_SCAN && tx.getVisibilityLevel() != Transaction.VisibilityLevel.SNAPSHOT_ALL); this.cellFilter = cellFilter; }
/** * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions. * * @param tx the current transaction to apply. Only data visible to this transaction will be returned. * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false} * these will be interpreted as "delete" markers and the column will be filtered out * @param scanType the type of scan operation being performed * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead. */ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType, @Nullable Filter cellFilter) { this.tx = tx; this.oldestTsByFamily = Maps.newTreeMap(); for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) { long familyTTL = ttlEntry.getValue(); oldestTsByFamily.put(new ImmutableBytesWritable(ttlEntry.getKey()), familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS); } this.allowEmptyValues = allowEmptyValues; this.clearDeletes = scanType == ScanType.COMPACT_DROP_DELETES || (scanType == ScanType.USER_SCAN && tx.getVisibilityLevel() != Transaction.VisibilityLevel.SNAPSHOT_ALL); this.cellFilter = cellFilter; }
/** * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions. * * @param tx the current transaction to apply. Only data visible to this transaction will be returned. * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false} * these will be interpreted as "delete" markers and the column will be filtered out * @param scanType the type of scan operation being performed * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead. */ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType, @Nullable Filter cellFilter) { this.tx = tx; this.oldestTsByFamily = Maps.newTreeMap(); for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) { long familyTTL = ttlEntry.getValue(); oldestTsByFamily.put(new ImmutableBytesWritable(ttlEntry.getKey()), familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS); } this.allowEmptyValues = allowEmptyValues; this.clearDeletes = scanType == ScanType.COMPACT_DROP_DELETES || (scanType == ScanType.USER_SCAN && tx.getVisibilityLevel() != Transaction.VisibilityLevel.SNAPSHOT_ALL); this.cellFilter = cellFilter; }
/** * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions. * * @param tx the current transaction to apply. Only data visible to this transaction will be returned. * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false} * these will be interpreted as "delete" markers and the column will be filtered out * @param scanType the type of scan operation being performed * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead. */ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType, @Nullable Filter cellFilter) { this.tx = tx; this.oldestTsByFamily = Maps.newTreeMap(); for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) { long familyTTL = ttlEntry.getValue(); oldestTsByFamily.put(new ImmutableBytesWritable(ttlEntry.getKey()), familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS); } this.allowEmptyValues = allowEmptyValues; this.clearDeletes = scanType == ScanType.COMPACT_DROP_DELETES || (scanType == ScanType.USER_SCAN && tx.getVisibilityLevel() != Transaction.VisibilityLevel.SNAPSHOT_ALL); this.cellFilter = cellFilter; }
/** * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions. * * @param tx the current transaction to apply. Only data visible to this transaction will be returned. * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false} * these will be interpreted as "delete" markers and the column will be filtered out * @param scanType the type of scan operation being performed * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead. */ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType, @Nullable Filter cellFilter) { this.tx = tx; this.oldestTsByFamily = Maps.newTreeMap(); for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) { long familyTTL = ttlEntry.getValue(); oldestTsByFamily.put(new ImmutableBytesWritable(ttlEntry.getKey()), familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS); } this.allowEmptyValues = allowEmptyValues; this.clearDeletes = scanType == ScanType.COMPACT_DROP_DELETES || (scanType == ScanType.USER_SCAN && tx.getVisibilityLevel() != Transaction.VisibilityLevel.SNAPSHOT_ALL); this.cellFilter = cellFilter; }
/** * Creates a new {@link org.apache.hadoop.hbase.filter.Filter} for returning data only from visible transactions. * * @param tx the current transaction to apply. Only data visible to this transaction will be returned. * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false} * these will be interpreted as "delete" markers and the column will be filtered out * @param scanType the type of scan operation being performed * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by * calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}. If null, then * {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead. */ public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType, @Nullable Filter cellFilter) { this.tx = tx; this.oldestTsByFamily = Maps.newTreeMap(); for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) { long familyTTL = ttlEntry.getValue(); oldestTsByFamily.put(new ImmutableBytesWritable(ttlEntry.getKey()), familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS); } this.allowEmptyValues = allowEmptyValues; this.clearDeletes = scanType == ScanType.COMPACT_DROP_DELETES || (scanType == ScanType.USER_SCAN && tx.getVisibilityLevel() != Transaction.VisibilityLevel.SNAPSHOT_ALL); this.cellFilter = cellFilter; }
private Transaction startTx(long expiration, TransactionType type) { Transaction tx = null; long txid; // guard against changes to the transaction log while processing this.logReadLock.lock(); try { synchronized (this) { ensureAvailable(); txid = getNextWritePointer(); tx = createTransaction(txid, type); addInProgressAndAdvance(tx.getTransactionId(), tx.getVisibilityUpperBound(), expiration, type); } // appending to WAL out of global lock for concurrent performance // we should still be able to arrive at the same state even if log entries are out of order appendToLog(TransactionEdit.createStarted(tx.getTransactionId(), tx.getVisibilityUpperBound(), expiration, type)); } finally { this.logReadLock.unlock(); } return tx; }
private Transaction startTx(long expiration, TransactionType type) { Transaction tx = null; long txid; // guard against changes to the transaction log while processing this.logReadLock.lock(); try { synchronized (this) { ensureAvailable(); txid = getNextWritePointer(); tx = createTransaction(txid, type); addInProgressAndAdvance(tx.getTransactionId(), tx.getVisibilityUpperBound(), expiration, type); } // appending to WAL out of global lock for concurrent performance // we should still be able to arrive at the same state even if log entries are out of order appendToLog(TransactionEdit.createStarted(tx.getTransactionId(), tx.getVisibilityUpperBound(), expiration, type)); } finally { this.logReadLock.unlock(); } return tx; }
protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action, TransactionVisibilityState snapshot, Store store, List<? extends KeyValueScanner> scanners, ScanType type, long earliestPutTs) throws IOException { if (snapshot == null) { if (LOG.isDebugEnabled()) { LOG.debug("Region " + env.getRegion().getRegionNameAsString() + ", no current transaction state found, defaulting to normal " + action + " scanner"); } return null; } // construct a dummy transaction from the latest snapshot Transaction dummyTx = TxUtils.createDummyTransaction(snapshot); Scan scan = new Scan(); // need to see all versions, since we filter out excludes and applications may rely on multiple versions scan.setMaxVersions(); scan.setFilter( new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(), snapshot.getInvalid(), getTransactionFilter(dummyTx, type, null))); return new StoreScanner(store, store.getScanInfo(), scan, scanners, type, store.getSmallestReadPoint(), earliestPutTs); }
protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action, TransactionVisibilityState snapshot, Store store, List<? extends KeyValueScanner> scanners, ScanType type, long earliestPutTs) throws IOException { if (snapshot == null) { if (LOG.isDebugEnabled()) { LOG.debug("Region " + env.getRegion().getRegionNameAsString() + ", no current transaction state found, defaulting to normal " + action + " scanner"); } return null; } // construct a dummy transaction from the latest snapshot Transaction dummyTx = TxUtils.createDummyTransaction(snapshot); Scan scan = new Scan(); // need to see all versions, since we filter out excludes and applications may rely on multiple versions scan.setMaxVersions(); scan.setFilter( new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(), snapshot.getInvalid(), getTransactionFilter(dummyTx, type, null))); return new StoreScanner(store, store.getScanInfo(), scan, scanners, type, store.getSmallestReadPoint(), earliestPutTs); }
protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action, TransactionVisibilityState snapshot, Store store, List<? extends KeyValueScanner> scanners, ScanType type, long earliestPutTs) throws IOException { if (snapshot == null) { if (LOG.isDebugEnabled()) { LOG.debug("Region " + env.getRegion().getRegionNameAsString() + ", no current transaction state found, defaulting to normal " + action + " scanner"); } return null; } // construct a dummy transaction from the latest snapshot Transaction dummyTx = TxUtils.createDummyTransaction(snapshot); Scan scan = new Scan(); // need to see all versions, since we filter out excludes and applications may rely on multiple versions scan.setMaxVersions(); scan.setFilter( new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(), snapshot.getInvalid(), getTransactionFilter(dummyTx, type, null))); return new StoreScanner(store, store.getScanInfo(), scan, scanners, type, store.getSmallestReadPoint(), earliestPutTs); }
protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action, TransactionVisibilityState snapshot, Store store, List<? extends KeyValueScanner> scanners, ScanType type, long earliestPutTs) throws IOException { if (snapshot == null) { if (LOG.isDebugEnabled()) { LOG.debug("Region " + env.getRegion().getRegionInfo().getRegionNameAsString() + ", no current transaction state found, defaulting to normal " + action + " scanner"); } return null; } // construct a dummy transaction from the latest snapshot Transaction dummyTx = TxUtils.createDummyTransaction(snapshot); Scan scan = new Scan(); // need to see all versions, since we filter out excludes and applications may rely on multiple versions scan.setMaxVersions(); scan.setFilter( new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(), snapshot.getInvalid(), getTransactionFilter(dummyTx, type, null))); return new StoreScanner(store, store.getScanInfo(), scan, scanners, type, store.getSmallestReadPoint(), earliestPutTs); }
protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action, TransactionVisibilityState snapshot, Store store, List<? extends KeyValueScanner> scanners, ScanType type, long earliestPutTs) throws IOException { if (snapshot == null) { if (LOG.isDebugEnabled()) { LOG.debug("Region " + env.getRegion().getRegionNameAsString() + ", no current transaction state found, defaulting to normal " + action + " scanner"); } return null; } // construct a dummy transaction from the latest snapshot Transaction dummyTx = TxUtils.createDummyTransaction(snapshot); Scan scan = new Scan(); // need to see all versions, since we filter out excludes and applications may rely on multiple versions scan.setMaxVersions(); scan.setFilter( new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(), snapshot.getInvalid(), getTransactionFilter(dummyTx, type, null))); return new StoreScanner(store, store.getScanInfo(), scan, scanners, type, store.getSmallestReadPoint(), earliestPutTs); }
protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action, TransactionVisibilityState snapshot, Store store, List<? extends KeyValueScanner> scanners, ScanType type, long earliestPutTs) throws IOException { if (snapshot == null) { if (LOG.isDebugEnabled()) { LOG.debug("Region " + env.getRegion().getRegionNameAsString() + ", no current transaction state found, defaulting to normal " + action + " scanner"); } return null; } // construct a dummy transaction from the latest snapshot Transaction dummyTx = TxUtils.createDummyTransaction(snapshot); Scan scan = new Scan(); // need to see all versions, since we filter out excludes and applications may rely on multiple versions scan.setMaxVersions(); scan.setFilter( new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(), snapshot.getInvalid(), getTransactionFilter(dummyTx, type, null))); return new StoreScanner(store, store.getScanInfo(), scan, scanners, type, store.getSmallestReadPoint(), earliestPutTs); }
protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action, TransactionVisibilityState snapshot, Store store, List<? extends KeyValueScanner> scanners, ScanType type, long earliestPutTs) throws IOException { if (snapshot == null) { if (LOG.isDebugEnabled()) { LOG.debug("Region " + env.getRegion().getRegionNameAsString() + ", no current transaction state found, defaulting to normal " + action + " scanner"); } return null; } // construct a dummy transaction from the latest snapshot Transaction dummyTx = TxUtils.createDummyTransaction(snapshot); Scan scan = new Scan(); // need to see all versions, since we filter out excludes and applications may rely on multiple versions scan.setMaxVersions(); scan.setFilter( new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(), snapshot.getInvalid(), getTransactionFilter(dummyTx, type, null))); return new StoreScanner(store, store.getScanInfo(), scan, scanners, type, store.getSmallestReadPoint(), earliestPutTs); }
protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action, TransactionVisibilityState snapshot, Store store, List<? extends KeyValueScanner> scanners, ScanType type, long earliestPutTs) throws IOException { if (snapshot == null) { if (LOG.isDebugEnabled()) { LOG.debug("Region " + env.getRegion().getRegionInfo().getRegionNameAsString() + ", no current transaction state found, defaulting to normal " + action + " scanner"); } return null; } // construct a dummy transaction from the latest snapshot Transaction dummyTx = TxUtils.createDummyTransaction(snapshot); Scan scan = new Scan(); // need to see all versions, since we filter out excludes and applications may rely on multiple versions scan.setMaxVersions(); scan.setFilter( new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(), snapshot.getInvalid(), getTransactionFilter(dummyTx, type, null))); return new StoreScanner(store, store.getScanInfo(), scan, scanners, type, store.getSmallestReadPoint(), earliestPutTs); }