Date: Wed, 1 Jun 2016 18:32:20 +0000 (UTC) From: Mateusz Guzik <mjg@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r301157 - in head/sys: kern sys Message-ID: <201606011832.u51IWK0Y078864@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: mjg Date: Wed Jun 1 18:32:20 2016 New Revision: 301157 URL: https://svnweb.freebsd.org/changeset/base/301157 Log: Microoptimize locking primitives by avoiding unnecessary atomic ops. Inline version of primitives do an atomic op and if it fails they fallback to actual primitives, which immediately retry the atomic op. The obvious optimisation is to check if the lock is free and only then proceed to do an atomic op. Reviewed by: jhb, vangyzen Modified: head/sys/kern/kern_lock.c head/sys/kern/kern_mutex.c head/sys/kern/kern_rwlock.c head/sys/kern/kern_sx.c head/sys/sys/mutex.h head/sys/sys/rwlock.h head/sys/sys/sx.h Modified: head/sys/kern/kern_lock.c ============================================================================== --- head/sys/kern/kern_lock.c Wed Jun 1 17:57:56 2016 (r301156) +++ head/sys/kern/kern_lock.c Wed Jun 1 18:32:20 2016 (r301157) @@ -787,8 +787,10 @@ __lockmgr_args(struct lock *lk, u_int fl break; } - while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, - tid)) { + for (;;) { + if (lk->lk_lock == LK_UNLOCKED && + atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) + break; #ifdef HWPMC_HOOKS PMC_SOFT_CALL( , , lock, failed); #endif @@ -1124,7 +1126,11 @@ __lockmgr_args(struct lock *lk, u_int fl __func__, iwmesg, file, line); } - while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) { + for (;;) { + if (lk->lk_lock == LK_UNLOCKED && + atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) + break; + #ifdef HWPMC_HOOKS PMC_SOFT_CALL( , , lock, failed); #endif Modified: head/sys/kern/kern_mutex.c ============================================================================== --- head/sys/kern/kern_mutex.c Wed Jun 1 17:57:56 2016 (r301156) +++ head/sys/kern/kern_mutex.c Wed Jun 1 18:32:20 2016 (r301157) @@ -419,7 +419,9 @@ __mtx_lock_sleep(volatile uintptr_t *c, all_time -= lockstat_nsecs(&m->lock_object); #endif - while (!_mtx_obtain_lock(m, tid)) { + for (;;) { + if (m->mtx_lock == MTX_UNOWNED && _mtx_obtain_lock(m, tid)) + break; #ifdef KDTRACE_HOOKS spin_cnt++; #endif @@ -602,8 +604,9 @@ _mtx_lock_spin_cookie(volatile uintptr_t #ifdef KDTRACE_HOOKS spin_time -= lockstat_nsecs(&m->lock_object); #endif - while (!_mtx_obtain_lock(m, tid)) { - + for (;;) { + if (m->mtx_lock == MTX_UNOWNED && _mtx_obtain_lock(m, tid)) + break; /* Give interrupts a chance while we spin. */ spinlock_exit(); while (m->mtx_lock != MTX_UNOWNED) { @@ -675,7 +678,9 @@ retry: m->lock_object.lo_name, file, line)); WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); - while (!_mtx_obtain_lock(m, tid)) { + for (;;) { + if (m->mtx_lock == MTX_UNOWNED && _mtx_obtain_lock(m, tid)) + break; if (m->mtx_lock == tid) { m->mtx_recurse++; break; Modified: head/sys/kern/kern_rwlock.c ============================================================================== --- head/sys/kern/kern_rwlock.c Wed Jun 1 17:57:56 2016 (r301156) +++ head/sys/kern/kern_rwlock.c Wed Jun 1 18:32:20 2016 (r301157) @@ -771,7 +771,9 @@ __rw_wlock_hard(volatile uintptr_t *c, u all_time -= lockstat_nsecs(&rw->lock_object); state = rw->rw_lock; #endif - while (!_rw_write_lock(rw, tid)) { + for (;;) { + if (rw->rw_lock == RW_UNLOCKED && _rw_write_lock(rw, tid)) + break; #ifdef KDTRACE_HOOKS spin_cnt++; #endif Modified: head/sys/kern/kern_sx.c ============================================================================== --- head/sys/kern/kern_sx.c Wed Jun 1 17:57:56 2016 (r301156) +++ head/sys/kern/kern_sx.c Wed Jun 1 18:32:20 2016 (r301157) @@ -544,7 +544,10 @@ _sx_xlock_hard(struct sx *sx, uintptr_t all_time -= lockstat_nsecs(&sx->lock_object); state = sx->sx_lock; #endif - while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) { + for (;;) { + if (sx->sx_lock == SX_LOCK_UNLOCKED && + atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) + break; #ifdef KDTRACE_HOOKS spin_cnt++; #endif Modified: head/sys/sys/mutex.h ============================================================================== --- head/sys/sys/mutex.h Wed Jun 1 17:57:56 2016 (r301156) +++ head/sys/sys/mutex.h Wed Jun 1 18:32:20 2016 (r301157) @@ -185,7 +185,7 @@ void thread_lock_flags_(struct thread *, #define __mtx_lock(mp, tid, opts, file, line) do { \ uintptr_t _tid = (uintptr_t)(tid); \ \ - if (!_mtx_obtain_lock((mp), _tid)) \ + if (((mp)->mtx_lock != MTX_UNOWNED || !_mtx_obtain_lock((mp), _tid)))\ _mtx_lock_sleep((mp), _tid, (opts), (file), (line)); \ else \ LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, \ @@ -203,7 +203,7 @@ void thread_lock_flags_(struct thread *, uintptr_t _tid = (uintptr_t)(tid); \ \ spinlock_enter(); \ - if (!_mtx_obtain_lock((mp), _tid)) { \ + if (((mp)->mtx_lock != MTX_UNOWNED || !_mtx_obtain_lock((mp), _tid))) {\ if ((mp)->mtx_lock == _tid) \ (mp)->mtx_recurse++; \ else \ @@ -232,7 +232,7 @@ void thread_lock_flags_(struct thread *, \ if ((mp)->mtx_recurse == 0) \ LOCKSTAT_PROFILE_RELEASE_LOCK(adaptive__release, mp); \ - if (!_mtx_release_lock((mp), _tid)) \ + if ((mp)->mtx_lock != _tid || !_mtx_release_lock((mp), _tid)) \ _mtx_unlock_sleep((mp), (opts), (file), (line)); \ } while (0) Modified: head/sys/sys/rwlock.h ============================================================================== --- head/sys/sys/rwlock.h Wed Jun 1 17:57:56 2016 (r301156) +++ head/sys/sys/rwlock.h Wed Jun 1 18:32:20 2016 (r301157) @@ -96,7 +96,7 @@ #define __rw_wlock(rw, tid, file, line) do { \ uintptr_t _tid = (uintptr_t)(tid); \ \ - if (!_rw_write_lock((rw), _tid)) \ + if ((rw)->rw_lock != RW_UNLOCKED || !_rw_write_lock((rw), _tid))\ _rw_wlock_hard((rw), _tid, (file), (line)); \ else \ LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, \ @@ -112,7 +112,7 @@ else { \ LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, \ LOCKSTAT_WRITER); \ - if (!_rw_write_unlock((rw), _tid)) \ + if ((rw)->rw_lock != _tid || !_rw_write_unlock((rw), _tid))\ _rw_wunlock_hard((rw), _tid, (file), (line)); \ } \ } while (0) Modified: head/sys/sys/sx.h ============================================================================== --- head/sys/sys/sx.h Wed Jun 1 17:57:56 2016 (r301156) +++ head/sys/sys/sx.h Wed Jun 1 18:32:20 2016 (r301157) @@ -150,7 +150,8 @@ __sx_xlock(struct sx *sx, struct thread uintptr_t tid = (uintptr_t)td; int error = 0; - if (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) + if (sx->sx_lock != SX_LOCK_UNLOCKED || + !atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) error = _sx_xlock_hard(sx, tid, opts, file, line); else LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx, @@ -168,7 +169,8 @@ __sx_xunlock(struct sx *sx, struct threa if (sx->sx_recurse == 0) LOCKSTAT_PROFILE_RELEASE_RWLOCK(sx__release, sx, LOCKSTAT_WRITER); - if (!atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED)) + if (sx->sx_lock != tid || + !atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED)) _sx_xunlock_hard(sx, tid, file, line); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201606011832.u51IWK0Y078864>