/** * @param ptr Pointer. * @param exp Expected. * @param v New value. * @return {@code true} If operation succeeded. */ public boolean casLong(long ptr, long exp, long v) { return GridUnsafe.compareAndSwapLong(null, ptr, exp, v); }
/** * @param ptr Pointer to update. * @param delta Delta. */ private static long updateAtomicLong(long ptr, long delta) { while (true) { long old = GridUnsafe.getLong(ptr); long updated = old + delta; if (GridUnsafe.compareAndSwapLong(null, ptr, old, updated)) return updated; } }
/** * Updates readers wait count. * * @param lock Lock to update. * @param delta Delta to update. */ private void updateReadersWaitCount(long lock, ReentrantLock lockObj, int delta) { assert lockObj.isHeldByCurrentThread(); while (true) { // Safe to do non-volatile read because of CAS below. long state = GridUnsafe.getLongVolatile(null, lock); long updated = updateState(state, 0, delta, 0); if (GridUnsafe.compareAndSwapLong(null, lock, state, updated)) return; } }
/** * Updates writers wait count. * * @param lock Lock to update. * @param delta Delta to update. */ private void updateWritersWaitCount(long lock, ReentrantLock lockObj, int delta) { assert lockObj.isHeldByCurrentThread(); while (true) { long state = GridUnsafe.getLongVolatile(null, lock); long updated = updateState(state, 0, 0, delta); if (GridUnsafe.compareAndSwapLong(null, lock, state, updated)) return; } } }
/** * @param relPtr Relative pointer to free. */ private void releaseFreePage(long relPtr) { long absPtr = absolute(relPtr); assert !PageHeader.isAcquired(absPtr) : "Release pinned page: " + PageHeader.fullPageId(absPtr); if (pagesCntr != null) pagesCntr.getAndDecrement(); while (true) { long freePageRelPtrMasked = GridUnsafe.getLong(freePageListPtr); long freePageRelPtr = freePageRelPtrMasked & RELATIVE_PTR_MASK; GridUnsafe.putLong(absPtr, freePageRelPtr); if (GridUnsafe.compareAndSwapLong(null, freePageListPtr, freePageRelPtrMasked, relPtr)) return; } }
/** * @param lock Lock address. */ public boolean tryWriteLock(long lock, int tag) { long state = GridUnsafe.getLongVolatile(null, lock); return checkTag(state, tag) && canWriteLock(state) && GridUnsafe.compareAndSwapLong(null, lock, state, updateState(state, -1, 0, 0)); }
if (GridUnsafe.compareAndSwapLong(null, lock, state, updated)) { int writeWaitCnt = writersWaitCount(updated); int readWaitCnt = readersWaitCount(updated); long updated = updateState(state, 1, -1, 0); if (GridUnsafe.compareAndSwapLong(null, lock, state, updated)) return true;
if (GridUnsafe.compareAndSwapLong(null, lock, state, updated)) { int writeWaitCnt = writersWaitCount(updated); int readWaitCnt = readersWaitCount(updated); long updated = updateState(state, -1, 0, -1); if (GridUnsafe.compareAndSwapLong(null, lock, state, updated)) return true;
/** * @param lock Lock address. */ public void readUnlock(long lock) { while (true) { long state = GridUnsafe.getLongVolatile(null, lock); if (lockCount(state) <= 0) throw new IllegalMonitorStateException("Attempted to release a read lock while not holding it " + "[lock=" + U.hexLong(lock) + ", state=" + U.hexLong(state) + ']'); long updated = updateState(state, -1, 0, 0); assert updated != 0; if (GridUnsafe.compareAndSwapLong(null, lock, state, updated)) { // Notify monitor if we were CASed to zero and there is a write waiter. if (lockCount(updated) == 0 && writersWaitCount(updated) > 0) { int idx = lockIndex(lock); ReentrantLock lockObj = locks[idx]; lockObj.lock(); try { // Note that we signal all waiters for this stripe. Since not all waiters in this // stripe/index belong to this particular lock, we can't wake up just one of them. writeConditions[idx].signalAll(); } finally { lockObj.unlock(); } } return; } } }
/** * @return Relative pointer to a free page that was borrowed from the allocated pool. */ private long borrowFreePage() { while (true) { long freePageRelPtrMasked = GridUnsafe.getLong(freePageListPtr); long freePageRelPtr = freePageRelPtrMasked & ADDRESS_MASK; if (freePageRelPtr != INVALID_REL_PTR) { long freePageAbsPtr = absolute(freePageRelPtr); long nextFreePageRelPtr = GridUnsafe.getLong(freePageAbsPtr) & ADDRESS_MASK; long cnt = ((freePageRelPtrMasked & COUNTER_MASK) + COUNTER_INC) & COUNTER_MASK; if (GridUnsafe.compareAndSwapLong(null, freePageListPtr, freePageRelPtrMasked, nextFreePageRelPtr | cnt)) { GridUnsafe.putLong(freePageAbsPtr, PAGE_MARKER); return freePageRelPtr; } } else return INVALID_REL_PTR; } }
if (GridUnsafe.compareAndSwapLong(null, lock, state, updateState(state, -2, 0, 0))) return true; else if (GridUnsafe.compareAndSwapLong(null, lock, state, updateState(state, -2, 0, 0))) return true; else if (GridUnsafe.compareAndSwapLong(null, lock, state, updateState(state, -1, 0, 1))) break;
/** * @param lock Lock address. */ public boolean writeLock(long lock, int tag) { assert tag != 0; for (int i = 0; i < SPIN_CNT; i++) { long state = GridUnsafe.getLongVolatile(null, lock); assert state != 0; if (!checkTag(state, tag)) return false; if (canWriteLock(state)) { if (GridUnsafe.compareAndSwapLong(null, lock, state, updateState(state, -1, 0, 0))) return true; else // Retry CAS, do not count as spin cycle. i--; } } int idx = lockIndex(lock); ReentrantLock lockObj = locks[idx]; lockObj.lock(); try { updateWritersWaitCount(lock, lockObj, 1); return waitAcquireWriteLock(lock, idx, tag); } finally { lockObj.unlock(); } }
/** * @param lock Lock address. */ public void writeUnlock(long lock, int tag) { long updated; assert tag != 0; while (true) { long state = GridUnsafe.getLongVolatile(null, lock); if (lockCount(state) != -1) throw new IllegalMonitorStateException("Attempted to release write lock while not holding it " + "[lock=" + U.hexLong(lock) + ", state=" + U.hexLong(state) + ']'); updated = releaseWithTag(state, tag); assert updated != 0; if (GridUnsafe.compareAndSwapLong(null, lock, state, updated)) break; } int writeWaitCnt = writersWaitCount(updated); int readWaitCnt = readersWaitCount(updated); if (writeWaitCnt > 0 || readWaitCnt > 0) { int idx = lockIndex(lock); ReentrantLock lockObj = locks[idx]; lockObj.lock(); try { signalNextWaiter(writeWaitCnt, readWaitCnt, idx); } finally { lockObj.unlock(); } } }
/** * @param tag Tag to initialize RW lock. * @return Relative pointer of the allocated page. * @throws GridOffHeapOutOfMemoryException If failed to allocate. */ private long allocateFreePage(int tag) throws GridOffHeapOutOfMemoryException { long limit = region.address() + region.size(); while (true) { long lastIdx = GridUnsafe.getLongVolatile(null, lastAllocatedIdxPtr); // Check if we have enough space to allocate a page. if (pagesBase + (lastIdx + 1) * sysPageSize > limit) return INVALID_REL_PTR; if (GridUnsafe.compareAndSwapLong(null, lastAllocatedIdxPtr, lastIdx, lastIdx + 1)) { long absPtr = pagesBase + lastIdx * sysPageSize; assert lastIdx <= PageIdUtils.MAX_PAGE_NUM : lastIdx; long pageIdx = fromSegmentIndex(idx, lastIdx); assert pageIdx != INVALID_REL_PTR; writePageId(absPtr, pageIdx); GridUnsafe.putLong(absPtr, PAGE_MARKER); rwLock.init(absPtr + LOCK_OFFSET, tag); allocatedPages.incrementAndGet(); memMetrics.updateTotalAllocatedPages(1L); return pageIdx; } } }
if (GridUnsafe.compareAndSwapLong(null, lock, state, updateState(state, 1, 0, 0))) return true; else
/** * @param pageId Page ID. * @return Relative pointer of the allocated page. * @throws GridOffHeapOutOfMemoryException If failed to allocate new free page. */ private long allocateFreePage(long pageId) throws GridOffHeapOutOfMemoryException { long limit = region.address() + region.size(); while (true) { long lastIdx = GridUnsafe.getLong(lastAllocatedIdxPtr); // Check if we have enough space to allocate a page. if (pagesBase + (lastIdx + 1) * sysPageSize > limit) return INVALID_REL_PTR; if (GridUnsafe.compareAndSwapLong(null, lastAllocatedIdxPtr, lastIdx, lastIdx + 1)) { long absPtr = pagesBase + lastIdx * sysPageSize; assert (lastIdx & SEGMENT_INDEX_MASK) == 0L; long relative = relative(lastIdx); assert relative != INVALID_REL_PTR; PageHeader.initNew(absPtr, relative); rwLock.init(absPtr + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId)); return relative; } } }
/** * @param ptr Pointer. * @param exp Expected. * @param v New value. * @return {@code true} If operation succeeded. */ public boolean casLong(long ptr, long exp, long v) { return GridUnsafe.compareAndSwapLong(null, ptr, exp, v); }
/** * @param ptr Pointer to update. * @param delta Delta. */ private static long updateAtomicLong(long ptr, long delta) { while (true) { long old = GridUnsafe.getLong(ptr); long updated = old + delta; if (GridUnsafe.compareAndSwapLong(null, ptr, old, updated)) return updated; } }
/** * Updates readers wait count. * * @param lock Lock to update. * @param delta Delta to update. */ private void updateReadersWaitCount(long lock, ReentrantLock lockObj, int delta) { assert lockObj.isHeldByCurrentThread(); while (true) { // Safe to do non-volatile read because of CAS below. long state = GridUnsafe.getLongVolatile(null, lock); long updated = updateState(state, 0, delta, 0); if (GridUnsafe.compareAndSwapLong(null, lock, state, updated)) return; } }
/** * @param lock Lock address. */ public boolean tryWriteLock(long lock, int tag) { long state = GridUnsafe.getLongVolatile(null, lock); return checkTag(state, tag) && canWriteLock(state) && GridUnsafe.compareAndSwapLong(null, lock, state, updateState(state, -1, 0, 0)); }