From owner-svn-src-stable@freebsd.org Thu Mar 16 06:56:25 2017 Return-Path: Delivered-To: svn-src-stable@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 2E871D0E4A8; Thu, 16 Mar 2017 06:56:25 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id E350115B7; Thu, 16 Mar 2017 06:56:24 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v2G6uObu090681; Thu, 16 Mar 2017 06:56:24 GMT (envelope-from mjg@FreeBSD.org) Received: (from mjg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v2G6uOFY090680; Thu, 16 Mar 2017 06:56:24 GMT (envelope-from mjg@FreeBSD.org) Message-Id: <201703160656.v2G6uOFY090680@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mjg set sender to mjg@FreeBSD.org using -f From: Mateusz Guzik Date: Thu, 16 Mar 2017 06:56:24 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-11@freebsd.org Subject: svn commit: r315381 - stable/11/sys/kern X-SVN-Group: stable-11 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-stable@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for all the -stable branches of the src tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 16 Mar 2017 06:56:25 -0000 Author: mjg Date: Thu Mar 16 06:56:23 2017 New Revision: 315381 URL: https://svnweb.freebsd.org/changeset/base/315381 Log: MFC r313455: sx: implement slock/sunlock fast path See r313454. Modified: stable/11/sys/kern/kern_sx.c Directory Properties: stable/11/ (props changed) Modified: stable/11/sys/kern/kern_sx.c ============================================================================== --- stable/11/sys/kern/kern_sx.c Thu Mar 16 06:53:55 2017 (r315380) +++ stable/11/sys/kern/kern_sx.c Thu Mar 16 06:56:23 2017 (r315381) @@ -797,8 +797,32 @@ _sx_xunlock_hard(struct sx *sx, uintptr_ kick_proc0(); } -int -_sx_slock(struct sx *sx, int opts, const char *file, int line) +static bool __always_inline +__sx_slock_try(struct sx *sx, uintptr_t *xp, const char *file, int line) +{ + + /* + * If no other thread has an exclusive lock then try to bump up + * the count of sharers. Since we have to preserve the state + * of SX_LOCK_EXCLUSIVE_WAITERS, if we fail to acquire the + * shared lock loop back and retry. + */ + while (*xp & SX_LOCK_SHARED) { + MPASS(!(*xp & SX_LOCK_SHARED_WAITERS)); + if (atomic_fcmpset_acq_ptr(&sx->sx_lock, xp, + *xp + SX_ONE_SHARER)) { + if (LOCK_LOG_TEST(&sx->lock_object, 0)) + CTR4(KTR_LOCK, "%s: %p succeed %p -> %p", + __func__, sx, (void *)*xp, + (void *)(*xp + SX_ONE_SHARER)); + return (true); + } + } + return (false); +} + +static int __noinline +_sx_slock_hard(struct sx *sx, int opts, const char *file, int line, uintptr_t x) { GIANT_DECLARE; #ifdef ADAPTIVE_SX @@ -808,7 +832,6 @@ _sx_slock(struct sx *sx, int opts, const uint64_t waittime = 0; int contested = 0; #endif - uintptr_t x; int error = 0; #if defined(ADAPTIVE_SX) || defined(KDTRACE_HOOKS) struct lock_delay_arg lda; @@ -828,17 +851,8 @@ _sx_slock(struct sx *sx, int opts, const #elif defined(KDTRACE_HOOKS) lock_delay_arg_init(&lda, NULL); #endif - KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), - ("sx_slock() by idle thread %p on sx %s @ %s:%d", - curthread, sx->lock_object.lo_name, file, line)); - KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, - ("sx_slock() of destroyed sx @ %s:%d", file, line)); - WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line, NULL); #ifdef KDTRACE_HOOKS all_time -= lockstat_nsecs(&sx->lock_object); -#endif - x = SX_READ_VALUE(sx); -#ifdef KDTRACE_HOOKS state = x; #endif @@ -847,25 +861,8 @@ _sx_slock(struct sx *sx, int opts, const * shared locks once there is an exclusive waiter. */ for (;;) { - /* - * If no other thread has an exclusive lock then try to bump up - * the count of sharers. Since we have to preserve the state - * of SX_LOCK_EXCLUSIVE_WAITERS, if we fail to acquire the - * shared lock loop back and retry. - */ - if (x & SX_LOCK_SHARED) { - MPASS(!(x & SX_LOCK_SHARED_WAITERS)); - if (atomic_fcmpset_acq_ptr(&sx->sx_lock, &x, - x + SX_ONE_SHARER)) { - if (LOCK_LOG_TEST(&sx->lock_object, 0)) - CTR4(KTR_LOCK, - "%s: %p succeed %p -> %p", __func__, - sx, (void *)x, - (void *)(x + SX_ONE_SHARER)); - break; - } - continue; - } + if (__sx_slock_try(sx, &x, file, line)) + break; #ifdef KDTRACE_HOOKS lda.spin_cnt++; #endif @@ -1004,51 +1001,62 @@ _sx_slock(struct sx *sx, int opts, const if (error == 0) { LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx, contested, waittime, file, line, LOCKSTAT_READER); - LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line); - WITNESS_LOCK(&sx->lock_object, 0, file, line); - TD_LOCKS_INC(curthread); } GIANT_RESTORE(); return (error); } -void -_sx_sunlock(struct sx *sx, const char *file, int line) +int +_sx_slock(struct sx *sx, int opts, const char *file, int line) { uintptr_t x; - int wakeup_swapper; - - if (SCHEDULER_STOPPED()) - return; + int error; + KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), + ("sx_slock() by idle thread %p on sx %s @ %s:%d", + curthread, sx->lock_object.lo_name, file, line)); KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, - ("sx_sunlock() of destroyed sx @ %s:%d", file, line)); - _sx_assert(sx, SA_SLOCKED, file, line); - WITNESS_UNLOCK(&sx->lock_object, 0, file, line); - LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line); - LOCKSTAT_PROFILE_RELEASE_RWLOCK(sx__release, sx, LOCKSTAT_READER); + ("sx_slock() of destroyed sx @ %s:%d", file, line)); + WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line, NULL); + + error = 0; x = SX_READ_VALUE(sx); + if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(sx__acquire) || + !__sx_slock_try(sx, &x, file, line))) + error = _sx_slock_hard(sx, opts, file, line, x); + if (error == 0) { + LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line); + WITNESS_LOCK(&sx->lock_object, 0, file, line); + TD_LOCKS_INC(curthread); + } + return (error); +} + +static bool __always_inline +_sx_sunlock_try(struct sx *sx, uintptr_t *xp) +{ + for (;;) { /* * We should never have sharers while at least one thread * holds a shared lock. */ - KASSERT(!(x & SX_LOCK_SHARED_WAITERS), + KASSERT(!(*xp & SX_LOCK_SHARED_WAITERS), ("%s: waiting sharers", __func__)); /* * See if there is more than one shared lock held. If * so, just drop one and return. */ - if (SX_SHARERS(x) > 1) { - if (atomic_fcmpset_rel_ptr(&sx->sx_lock, &x, - x - SX_ONE_SHARER)) { + if (SX_SHARERS(*xp) > 1) { + if (atomic_fcmpset_rel_ptr(&sx->sx_lock, xp, + *xp - SX_ONE_SHARER)) { if (LOCK_LOG_TEST(&sx->lock_object, 0)) CTR4(KTR_LOCK, "%s: %p succeeded %p -> %p", - __func__, sx, (void *)x, - (void *)(x - SX_ONE_SHARER)); - break; + __func__, sx, (void *)*xp, + (void *)(*xp - SX_ONE_SHARER)); + return (true); } continue; } @@ -1057,18 +1065,36 @@ _sx_sunlock(struct sx *sx, const char *f * If there aren't any waiters for an exclusive lock, * then try to drop it quickly. */ - if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) { - MPASS(x == SX_SHARERS_LOCK(1)); - x = SX_SHARERS_LOCK(1); + if (!(*xp & SX_LOCK_EXCLUSIVE_WAITERS)) { + MPASS(*xp == SX_SHARERS_LOCK(1)); + *xp = SX_SHARERS_LOCK(1); if (atomic_fcmpset_rel_ptr(&sx->sx_lock, - &x, SX_LOCK_UNLOCKED)) { + xp, SX_LOCK_UNLOCKED)) { if (LOCK_LOG_TEST(&sx->lock_object, 0)) CTR2(KTR_LOCK, "%s: %p last succeeded", __func__, sx); - break; + return (true); } continue; } + break; + } + return (false); +} + +static void __noinline +_sx_sunlock_hard(struct sx *sx, uintptr_t x, const char *file, int line) +{ + int wakeup_swapper; + + if (SCHEDULER_STOPPED()) + return; + + LOCKSTAT_PROFILE_RELEASE_RWLOCK(sx__release, sx, LOCKSTAT_READER); + + for (;;) { + if (_sx_sunlock_try(sx, &x)) + break; /* * At this point, there should just be one sharer with @@ -1101,6 +1127,24 @@ _sx_sunlock(struct sx *sx, const char *f kick_proc0(); break; } +} + +void +_sx_sunlock(struct sx *sx, const char *file, int line) +{ + uintptr_t x; + + KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, + ("sx_sunlock() of destroyed sx @ %s:%d", file, line)); + _sx_assert(sx, SA_SLOCKED, file, line); + WITNESS_UNLOCK(&sx->lock_object, 0, file, line); + LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line); + + x = SX_READ_VALUE(sx); + if (__predict_false(LOCKSTAT_OOL_PROFILE_ENABLED(sx__release) || + !_sx_sunlock_try(sx, &x))) + _sx_sunlock_hard(sx, x, file, line); + TD_LOCKS_DEC(curthread); }