private Transaction getReadOnlyTransaction(final long commitTs) { return new SnapshotTransaction( metricsManager, keyValueService,
@Override public void commit() { commit(defaultTransactionService); }
@Override @Idempotent public Map<Cell, byte[]> get(TableReference tableRef, Set<Cell> cells) { Map<Cell, byte[]> ret = super.get(tableRef, cells); markCellsRead(tableRef, cells, ret); return ret; }
private void scrubForAggressiveHardDelete(SnapshotTransaction tx) { if ((tx.getTransactionType() == TransactionType.AGGRESSIVE_HARD_DELETE) && !tx.isAborted()) { // t.getCellsToScrubImmediately() checks that t has been committed cleaner.scrubImmediately(this, tx.getCellsToScrubImmediately(), tx.getTimestamp(), tx.getCommitTimestamp()); } }
@Override public Map<Cell, byte[]> getIgnoringLocalWrites(TableReference tableRef, Set<Cell> cells) { checkGetPreconditions(tableRef); if (Iterables.isEmpty(cells)) { return ImmutableMap.of(); } hasReads = true; Map<Cell, byte[]> result = getFromKeyValueService(tableRef, cells); validatePreCommitRequirementsOnReadIfNecessary(tableRef, getStartTimestamp()); return Maps.filterValues(result, Predicates.not(Value.IS_EMPTY)); }
@Override public SortedMap<byte[], RowResult<byte[]>> getRowsIgnoringLocalWrites( TableReference tableRef, Iterable<byte[]> rows) { checkGetPreconditions(tableRef); if (Iterables.isEmpty(rows)) { return AbstractTransaction.EMPTY_SORTED_ROWS; } hasReads = true; Map<Cell, Value> rawResults = Maps.newHashMap(keyValueService.getRows(tableRef, rows, ColumnSelection.all(), getStartTimestamp())); validatePreCommitRequirementsOnReadIfNecessary(tableRef, getStartTimestamp()); return filterRowResults(tableRef, rawResults, ImmutableMap.builderWithExpectedSize(rawResults.size())); }
private void commitWrites(TransactionService transactionService) { if (!hasWrites()) { if (hasReads()) { preCommitCondition.throwIfConditionInvalid(getStartTimestamp()); if (validationNecessaryForInvolvedTablesOnCommit()) { throwIfImmutableTsOrCommitLocksExpired(null); Timer.Context commitStageTimer = getTimer("commitStage").time(); Timer.Context acquireLocksTimer = getTimer("commitAcquireLocks").time(); LockToken commitLocksToken = acquireLocksForCommit(); long microsForRowLocks = TimeUnit.NANOSECONDS.toMicros(acquireLocksTimer.stop()); try { long microsCheckingForConflicts = runAndReportTimeAndGetDurationMicros( () -> throwIfConflictOnCommit(commitLocksToken, transactionService), "commitCheckingForConflicts"); runAndReportTimeAndGetDurationMicros(() -> sweepQueue.enqueue(writesByTable, getStartTimestamp()), "writingToSweepQueue"); long microsForWrites = runAndReportTimeAndGetDurationMicros( () -> keyValueService.multiPut(writesByTable, getStartTimestamp()), "commitWrite"); Timer.Context commitTimestampTimer = getTimer("getCommitTimestamp").time(); long commitTimestamp = timelockService.getFreshTimestamp(); commitTsForScrubbing = commitTimestamp;
@Override public Iterable<BatchingVisitable<RowResult<byte[]>>> getRanges(final TableReference tableRef, Iterable<RangeRequest> rangeRequests) { checkGetPreconditions(tableRef); Timer.Context timer = getTimer("processedRangeMillis").time(); Map<RangeRequest, TokenBackedBasicResultsPage<RowResult<Value>, byte[]>> firstPages = keyValueService.getFirstBatchForRanges(tableRef, input, getStartTimestamp()); validatePreCommitRequirementsOnReadIfNecessary(tableRef, getStartTimestamp()); SortedMap<Cell, byte[]> postFiltered = postFilterPages( tableRef, firstPages.values()); TokenBackedBasicResultsPage<RowResult<Value>, byte[]> prePostFilter = firstPages.get(rangeRequest); byte[] nextStartRowName = getNextStartRowName( rangeRequest, prePostFilter); List<Entry<Cell, byte[]>> mergeIterators = getPostFilteredWithLocalWrites( tableRef, postFiltered,
@Override public SortedMap<byte[], RowResult<byte[]>> getRows(TableReference tableRef, Iterable<byte[]> rows, ColumnSelection columnSelection) { Timer.Context timer = getTimer("getRows").time(); checkGetPreconditions(tableRef); if (Iterables.isEmpty(rows)) { return AbstractTransaction.EMPTY_SORTED_ROWS; } hasReads = true; ImmutableMap.Builder<Cell, byte[]> result = ImmutableSortedMap.naturalOrder(); Map<Cell, Value> rawResults = Maps.newHashMap( keyValueService.getRows(tableRef, rows, columnSelection, getStartTimestamp())); SortedMap<Cell, byte[]> writes = writesByTable.get(tableRef); if (writes != null) { for (byte[] row : rows) { extractLocalWritesForRow(result, writes, row, columnSelection); } } // We don't need to do work postFiltering if we have a write locally. rawResults.keySet().removeAll(result.build().keySet()); SortedMap<byte[], RowResult<byte[]>> results = filterRowResults(tableRef, rawResults, result); long getRowsMillis = TimeUnit.NANOSECONDS.toMillis(timer.stop()); if (perfLogger.isDebugEnabled()) { perfLogger.debug("getRows({}, {} rows) found {} rows, took {} ms", tableRef, Iterables.size(rows), results.size(), getRowsMillis); } validatePreCommitRequirementsOnReadIfNecessary(tableRef, getStartTimestamp()); return results; }
ensureUncommitted(); if (state.compareAndSet(State.UNCOMMITTED, State.COMMITTING)) { break; if (getTransactionType() == TransactionType.AGGRESSIVE_HARD_DELETE || getTransactionType() == TransactionType.HARD_DELETE) { cleaner.queueCellsForScrubbing(getCellsToQueueForScrubbing(), getStartTimestamp()); checkConstraints(); commitWrites(transactionService); if (perfLogger.isDebugEnabled()) { long transactionMillis = TimeUnit.NANOSECONDS.toMillis(transactionTimerContext.stop()); perfLogger.debug("Committed transaction {} in {}ms", getStartTimestamp(), transactionMillis);
@Override public Iterator<Map.Entry<Cell, byte[]>> getRowsColumnRange(TableReference tableRef, Iterable<byte[]> rows, ColumnRangeSelection columnRangeSelection, int batchHint) { checkGetPreconditions(tableRef); if (Iterables.isEmpty(rows)) { return Collections.emptyIterator(); } hasReads = true; RowColumnRangeIterator rawResults = keyValueService.getRowsColumnRange(tableRef, rows, columnRangeSelection, batchHint, getStartTimestamp()); if (!rawResults.hasNext()) { validatePreCommitRequirementsOnReadIfNecessary(tableRef, getStartTimestamp()); } // else the postFiltered iterator will check for each batch. Iterator<Map.Entry<byte[], RowColumnRangeIterator>> rawResultsByRow = partitionByRow(rawResults); Iterator<Iterator<Map.Entry<Cell, byte[]>>> postFiltered = Iterators.transform(rawResultsByRow, e -> { byte[] row = e.getKey(); RowColumnRangeIterator rawIterator = e.getValue(); BatchColumnRangeSelection batchColumnRangeSelection = BatchColumnRangeSelection.create(columnRangeSelection, batchHint); return getPostFilteredColumns(tableRef, batchColumnRangeSelection, row, rawIterator); }); return Iterators.concat(postFiltered); }
@Test public void checkImmutableTsLockOnceIfThoroughlySwept_WithValidationOnReads() { TimelockService timelockService = spy(new LegacyTimelockService(timestampService, lockService, lockClient)); long transactionTs = timelockService.getFreshTimestamp(); LockImmutableTimestampResponse res = timelockService.lockImmutableTimestamp(IdentifiedTimeLockRequest.create()); SnapshotTransaction transaction = getSnapshotTransactionWith( timelockService, () -> transactionTs, res, PreCommitConditions.NO_OP, true); transaction.get(TABLE_SWEPT_THOROUGH, ImmutableSet.of(TEST_CELL)); transaction.commit(); timelockService.unlock(ImmutableSet.of(res.getLock())); verify(timelockService).refreshLockLeases(ImmutableSet.of(res.getLock())); }
Map<Cell, byte[]> oldValues = getIgnoringLocalWrites(table, cellToTs.keySet()); Map<Cell, Value> conflictingValues = keyValueService.get(table, cellToTs); throwIfPreCommitRequirementsNotMet(commitLocksToken, getStartTimestamp()); Validate.isTrue(false, "Missing conflicting value for cell: %s for table %s", cellToConflict.get(cell), table); throwIfPreCommitRequirementsNotMet(commitLocksToken, getStartTimestamp()); Validate.isTrue(false, "Wrong timestamp for cell in table %s Expected: %s Actual: %s", table, cellToConflict.get(cell), transactionOutcomeMetrics.markWriteWriteConflict(table); throw TransactionConflictException.create(table, getStartTimestamp(), Sets.filter(spanningWrites, conflicting), Sets.filter(dominatingWrites, conflicting),
@Override public Map<byte[], BatchingVisitable<Map.Entry<Cell, byte[]>>> getRowsColumnRange( TableReference tableRef, Iterable<byte[]> rows, BatchColumnRangeSelection columnRangeSelection) { checkGetPreconditions(tableRef); if (Iterables.isEmpty(rows)) { return ImmutableMap.of(); } hasReads = true; Map<byte[], RowColumnRangeIterator> rawResults = keyValueService.getRowsColumnRange(tableRef, rows, columnRangeSelection, getStartTimestamp()); Map<byte[], BatchingVisitable<Map.Entry<Cell, byte[]>>> postFilteredResults = Maps.newHashMapWithExpectedSize(rawResults.size()); for (Entry<byte[], RowColumnRangeIterator> e : rawResults.entrySet()) { byte[] row = e.getKey(); RowColumnRangeIterator rawIterator = e.getValue(); Iterator<Map.Entry<Cell, byte[]>> postFilteredIterator = getPostFilteredColumns(tableRef, columnRangeSelection, row, rawIterator); postFilteredResults.put(row, BatchingVisitableFromIterable.create(postFilteredIterator)); } return postFilteredResults; }
@Test public void commitDoesNotThrowIfAlreadySuccessfullyCommitted() { final Cell cell = Cell.create(PtBytes.toBytes("row1"), PtBytes.toBytes("column1")); TimestampService timestampServiceSpy = spy(timestampService); TimelockService timelockService = new LegacyTimelockService(timestampServiceSpy, lockService, lockClient); long transactionTs = timelockService.getFreshTimestamp(); LockImmutableTimestampResponse res = timelockService.lockImmutableTimestamp(IdentifiedTimeLockRequest.create()); SnapshotTransaction snapshot = getSnapshotTransactionWith( timelockService, () -> transactionTs, res, PreCommitConditions.NO_OP); when(timestampServiceSpy.getFreshTimestamp()).thenReturn(10000000L); //forcing to try to commit a transaction that is already committed transactionService.putUnlessExists(transactionTs, timelockService.getFreshTimestamp()); snapshot.put(TABLE, ImmutableMap.of(cell, PtBytes.toBytes("value"))); snapshot.commit(); timelockService.unlock(Collections.singleton(res.getLock())); }
@Test public void commitThrowsIfRolledBackAtCommitTime_expiredLocks() { final Cell cell = Cell.create(PtBytes.toBytes("row1"), PtBytes.toBytes("column1")); TimelockService timelockService = spy(new LegacyTimelockService(timestampService, lockService, lockClient)); // expire the locks when the pre-commit check happens - this is guaranteed to be after we've written the data PreCommitCondition condition = unused -> doReturn(ImmutableSet.of()).when(timelockService).refreshLockLeases(any()); LockImmutableTimestampResponse res = timelockService.lockImmutableTimestamp(IdentifiedTimeLockRequest.create()); long transactionTs = timelockService.getFreshTimestamp(); SnapshotTransaction snapshot = getSnapshotTransactionWith( timelockService, () -> transactionTs, res, condition); //simulate roll back at commit time transactionService.putUnlessExists(snapshot.getTimestamp(), TransactionConstants.FAILED_COMMIT_TS); snapshot.put(TABLE, ImmutableMap.of(cell, PtBytes.toBytes("value"))); assertThatExceptionOfType(TransactionLockTimeoutException.class).isThrownBy(snapshot::commit); timelockService.unlock(ImmutableSet.of(res.getLock())); TransactionOutcomeMetricsAssert.assertThat(transactionOutcomeMetrics) .hasFailedCommits(1) .hasLocksExpired(1); }
(t, heldLocks) -> { SnapshotTransaction snapshotTx = unwrapSnapshotTransaction(t); snapshotTx.getRowsIgnoringLocalWrites( TABLE_SWEPT_THOROUGH, Collections.singleton(PtBytes.toBytes("row1"))); (t, heldLocks) -> { SnapshotTransaction snapshotTx = unwrapSnapshotTransaction(t); snapshotTx.getIgnoringLocalWrites(TABLE_SWEPT_THOROUGH, Collections.singleton(Cell.create(PtBytes.toBytes("row1"), PtBytes.toBytes("column1")))); return null;
@Override @Idempotent public void put(TableReference tableRef, Map<Cell, byte[]> values) { super.put(tableRef, values); }
@Override public BatchingVisitable<RowResult<byte[]>> getRange(final TableReference tableRef, final RangeRequest range) { checkGetPreconditions(tableRef); if (range.isEmptyRange()) { return BatchingVisitables.emptyBatchingVisitable(); } hasReads = true; return new AbstractBatchingVisitable<RowResult<byte[]>>() { @Override public <K extends Exception> void batchAcceptSizeHint( int userRequestedSize, ConsistentVisitor<RowResult<byte[]>, K> visitor) throws K { ensureUncommitted(); int requestSize = range.getBatchHint() != null ? range.getBatchHint() : userRequestedSize; int preFilterBatchSize = getRequestHintToKvStore(requestSize); Validate.isTrue(!range.isReverse(), "we currently do not support reverse ranges"); getBatchingVisitableFromIterator( tableRef, range, requestSize, visitor, preFilterBatchSize); } }; }