@Override public LockResponse lock(LockRequest request) { return executeWithRecord(() -> timelockService.lock(request)); }
@Override protected void performOneCall() { LockToken token = timelock.lock(nextRequest()).getToken(); boolean wasUnlocked = timelock.unlock(ImmutableSet.of(token)).contains(token); Preconditions.checkState(wasUnlocked, "unlock returned false"); }
@Override public LockToken lock(String client, String lockName) throws InterruptedException { LockRequest lockRequest = LockRequest.of( ImmutableSet.of(StringLockDescriptor.of(lockName)), Long.MAX_VALUE, client); LockResponse lockResponse = timelockService.lock(lockRequest); Preconditions.checkState(lockResponse.wasSuccessful(), "Jepsen failed to lock a lock, but it would wait for Long.MAX_VALUE, so this is unexpected."); return lockResponse.getToken(); }
public static Optional<TargetedSweeperLock> tryAcquire(int shard, TableMetadataPersistence.SweepStrategy strategy, TimelockService timeLock) { ShardAndStrategy shardStrategy = ShardAndStrategy.of(shard, strategy); LockDescriptor lock = StringLockDescriptor.of(shardStrategy.toText()); // We do not want the timeout to be too low to avoid a race condition where we give up too soon LockRequest request = LockRequest.of(ImmutableSet.of(lock), 100L); return timeLock.lock(request) .getTokenOrEmpty() .map(lockToken -> new TargetedSweeperLock(shardStrategy, timeLock, lockToken)); }
@Test public void unsuccessfulLock() throws InterruptedException { when(mockLockService.lock(any())).thenReturn(() -> Optional.empty()); Optional<TargetedSweeperLock> maybeLock = TargetedSweeperLock .tryAcquire(2, TableMetadataPersistence.SweepStrategy.THOROUGH, mockLockService); assertThat(maybeLock).isNotPresent(); verify(mockLockService, times(1)).lock(any()); verifyNoMoreInteractions(mockLockService); } }
@Override public LockResponse lock(LockRequest request) { LockResponse response = executeOnTimeLock(() -> delegate.lock(request)); if (response.wasSuccessful()) { lockRefresher.registerLock(response.getToken()); } return response; }
/** * Creates a mock of a LockService that only gives out a lock once per unique request and never releases it, even * if unlock is called. The returned tokens are monotonically increasing in the tokenId. */ private TimelockService createStickyLockService() { AtomicLong lockToken = new AtomicLong(0); Set<LockDescriptor> requestedLocks = new ConcurrentHashSet<>(); TimelockService stickyLockService = mock(TimelockService.class); doAnswer(invocation -> { LockRequest request = invocation.getArgument(0); if (requestedLocks.add(Iterables.getOnlyElement(request.getLockDescriptors()))) { return (LockResponse) () -> Optional.of(LockToken.of(new UUID(lockToken.getAndIncrement(), 0L))); } else { return (LockResponse) Optional::empty; } }).when(stickyLockService).lock(any()); return stickyLockService; }
@Test public void successfulLockAndUnlock() throws InterruptedException { LockToken lockToken = LockToken.of(UUID.randomUUID()); when(mockLockService.lock(any())) .thenReturn(() -> Optional.of(lockToken)); Optional<TargetedSweeperLock> maybeLock = TargetedSweeperLock .tryAcquire(1, TableMetadataPersistence.SweepStrategy.CONSERVATIVE, mockLockService); assertThat(maybeLock).isPresent(); TargetedSweeperLock lock = maybeLock.get(); assertThat(lock.getShardAndStrategy()).isEqualTo(ShardAndStrategy.conservative(1)); lock.unlock(); verify(mockLockService, times(1)).unlock(ImmutableSet.of(lockToken)); verify(mockLockService, times(1)).lock(any()); verifyNoMoreInteractions(mockLockService); }
@Test public void extraSweepersGiveUpAfterFailingToAcquireEnoughTimes() throws InterruptedException { int shards = 16; int sweepers = 4; int threads = shards / (sweepers / 2); TimelockService stickyLockService = createStickyLockService(); createAndInitializeSweepersAndWaitForOneBackgroundIteration(sweepers, shards, threads, stickyLockService); ArgumentCaptor<LockRequest> captor = ArgumentCaptor.forClass(LockRequest.class); // minimum: as in the example above, but we have extra threads // threads + ... + threads * (shards / threads) + shards * (threads * sweepers - shards) verify(stickyLockService, atLeast(shards * (shards / threads + 1) / 2 + shards * (threads * sweepers - shards))) .lock(captor.capture()); // maximum: one would think that it is // shards + shards - 1 + ... + shards - (sweepers - 1) + shards * (threads * sweepers - shards) // but actually the logic is much more complicated since threads from the same sweeper can loop back and hit a // race condition with each other, so we go with the more conservative upper bound verify(stickyLockService, atMost(threads * sweepers * shards)).lock(any()); Set<String> requestedLockIds = captor.getAllValues().stream() .map(LockRequest::getLockDescriptors) .map(Iterables::getOnlyElement) .map(LockDescriptor::getLockIdAsString) .collect(Collectors.toSet()); Set<String> expectedLockIds = IntStream.range(0, shards).boxed() .map(ShardAndStrategy::conservative) .map(ShardAndStrategy::toText) .collect(Collectors.toSet()); assertThat(requestedLockIds).hasSameElementsAs(expectedLockIds); }
/** * This method should acquire any locks needed to do proper concurrency control at commit time. */ protected LockToken acquireLocksForCommit() { Set<LockDescriptor> lockDescriptors = getLocksForWrites(); LockRequest request = LockRequest.of(lockDescriptors, transactionConfig.get().getLockAcquireTimeoutMillis()); LockResponse lockResponse = timelockService.lock(request); if (!lockResponse.wasSuccessful()) { log.error("Timed out waiting while acquiring commit locks. Request id was {}. Timeout was {} ms. " + "First ten required locks were {}.", SafeArg.of("requestId", request.getRequestId()), SafeArg.of("acquireTimeoutMs", transactionConfig.get().getLockAcquireTimeoutMillis()), UnsafeArg.of("firstTenLockDescriptors", Iterables.limit(lockDescriptors, 10))); throw new TransactionLockAcquisitionTimeoutException("Timed out while acquiring commit locks."); } return lockResponse.getToken(); }
@Test public void multipleSweepersSweepDifferentShardsAndCallUnlockAfterwards() throws InterruptedException { int shards = 128; int sweepers = 8; int threads = shards / sweepers; TimelockService stickyLockService = createStickyLockService(); createAndInitializeSweepersAndWaitForOneBackgroundIteration(sweepers, shards, threads, stickyLockService); for (int i = 0; i < shards; i++) { assertProgressUpdatedToTimestamp(maxTsForFinePartition(tsPartitionFine(unreadableTs - 1)), i); verify(stickyLockService, times(1)).unlock(ImmutableSet.of(LockToken.of(new UUID(i, 0L)))); } // minimum: all threads on one host succeed, then on another, etc: // threads + threads * 2 + ... + threads * swepers verify(stickyLockService, atLeast(threads * sweepers * (sweepers - 1) / 2)) .lock(any(LockRequest.class)); // maximum: all but one succeed on each host, and only then those succeed: // shards + shards - 1 + ... + shards - (sweepers - 1) verify(stickyLockService, atMost(sweepers * shards - sweepers * (sweepers - 1) / 2)) .lock(any(LockRequest.class)); }
@Override public LockResponse lock(LockRequest request) { return executeWithRecord(() -> timelockService.lock(request)); }
public static Optional<TargetedSweeperLock> tryAcquire(int shard, TableMetadataPersistence.SweepStrategy strategy, TimelockService timeLock) { ShardAndStrategy shardStrategy = ShardAndStrategy.of(shard, strategy); LockDescriptor lock = StringLockDescriptor.of(shardStrategy.toText()); // We do not want the timeout to be too low to avoid a race condition where we give up too soon LockRequest request = LockRequest.of(ImmutableSet.of(lock), 100L); return timeLock.lock(request) .getTokenOrEmpty() .map(lockToken -> new TargetedSweeperLock(shardStrategy, timeLock, lockToken)); }
@Override public LockResponse lock(LockRequest request) { LockResponse response = executeOnTimeLock(() -> delegate.lock(request)); if (response.wasSuccessful()) { lockRefresher.registerLock(response.getToken()); } return response; }
/** * This method should acquire any locks needed to do proper concurrency control at commit time. */ protected LockToken acquireLocksForCommit() { Set<LockDescriptor> lockDescriptors = getLocksForWrites(); LockRequest request = LockRequest.of(lockDescriptors, transactionConfig.get().getLockAcquireTimeoutMillis()); LockResponse lockResponse = timelockService.lock(request); if (!lockResponse.wasSuccessful()) { log.error("Timed out waiting while acquiring commit locks. Request id was {}. Timeout was {} ms. " + "First ten required locks were {}.", SafeArg.of("requestId", request.getRequestId()), SafeArg.of("acquireTimeoutMs", transactionConfig.get().getLockAcquireTimeoutMillis()), UnsafeArg.of("firstTenLockDescriptors", Iterables.limit(lockDescriptors, 10))); throw new TransactionLockAcquisitionTimeoutException("Timed out while acquiring commit locks."); } return lockResponse.getToken(); }