From owner-svn-src-all@freebsd.org Sun Feb 5 13:37:25 2017 Return-Path: Delivered-To: svn-src-all@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 744FFCD1B7D; Sun, 5 Feb 2017 13:37:25 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 3E54618D0; Sun, 5 Feb 2017 13:37:25 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v15DbO0V088957; Sun, 5 Feb 2017 13:37:24 GMT (envelope-from mjg@FreeBSD.org) Received: (from mjg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v15DbOfU088955; Sun, 5 Feb 2017 13:37:24 GMT (envelope-from mjg@FreeBSD.org) Message-Id: <201702051337.v15DbOfU088955@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mjg set sender to mjg@FreeBSD.org using -f From: Mateusz Guzik Date: Sun, 5 Feb 2017 13:37:24 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r313282 - in head/sys: kern sys X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 05 Feb 2017 13:37:25 -0000 Author: mjg Date: Sun Feb 5 13:37:23 2017 New Revision: 313282 URL: https://svnweb.freebsd.org/changeset/base/313282 Log: rwlock: move lockstat handling out of inline primitives See r313275 for details. One difference here is that recursion handling was removed from the fallback routine. As it is it was never supposed to see a recursed lock in the first place. Future changes will move it out of inline variants, but right now there is no easy to way to test if the lock is recursed without reading additional words. Modified: head/sys/kern/kern_rwlock.c head/sys/sys/rwlock.h Modified: head/sys/kern/kern_rwlock.c ============================================================================== --- head/sys/kern/kern_rwlock.c Sun Feb 5 13:24:54 2017 (r313281) +++ head/sys/kern/kern_rwlock.c Sun Feb 5 13:37:23 2017 (r313282) @@ -283,6 +283,7 @@ void _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line) { struct rwlock *rw; + uintptr_t tid, v; if (SCHEDULER_STOPPED()) return; @@ -296,7 +297,14 @@ _rw_wlock_cookie(volatile uintptr_t *c, ("rw_wlock() of destroyed rwlock @ %s:%d", file, line)); WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); - __rw_wlock(rw, curthread, file, line); + tid = (uintptr_t)curthread; + v = RW_UNLOCKED; + if (!_rw_write_lock_fetch(rw, &v, tid)) + _rw_wlock_hard(rw, v, tid, file, line); + else + LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, + 0, 0, file, line, LOCKSTAT_WRITER); + LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line); WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); TD_LOCKS_INC(curthread); @@ -355,7 +363,11 @@ _rw_wunlock_cookie(volatile uintptr_t *c WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line); - __rw_wunlock(rw, curthread, file, line); + if (rw->rw_recurse) + rw->rw_recurse--; + else + _rw_wunlock_hard(rw, (uintptr_t)curthread, file, line); + TD_LOCKS_DEC(curthread); } @@ -998,13 +1010,12 @@ __rw_wunlock_hard(volatile uintptr_t *c, return; rw = rwlock2rw(c); + MPASS(!rw_recursed(rw)); - if (rw_wlocked(rw) && rw_recursed(rw)) { - rw->rw_recurse--; - if (LOCK_LOG_TEST(&rw->lock_object, 0)) - CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw); + LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, + LOCKSTAT_WRITER); + if (_rw_write_unlock(rw, tid)) return; - } KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), ("%s: neither of the waiter flags are set", __func__)); Modified: head/sys/sys/rwlock.h ============================================================================== --- head/sys/sys/rwlock.h Sun Feb 5 13:24:54 2017 (r313281) +++ head/sys/sys/rwlock.h Sun Feb 5 13:37:23 2017 (r313282) @@ -84,10 +84,8 @@ #define _rw_write_lock(rw, tid) \ atomic_cmpset_acq_ptr(&(rw)->rw_lock, RW_UNLOCKED, (tid)) -#define _rw_write_lock_fetch(rw, vp, tid) ({ \ - *vp = RW_UNLOCKED; \ - atomic_fcmpset_acq_ptr(&(rw)->rw_lock, vp, (tid)); \ -}) +#define _rw_write_lock_fetch(rw, vp, tid) \ + atomic_fcmpset_acq_ptr(&(rw)->rw_lock, vp, (tid)) /* Release a write lock quickly if there are no waiters. */ #define _rw_write_unlock(rw, tid) \ @@ -102,13 +100,11 @@ /* Acquire a write lock. */ #define __rw_wlock(rw, tid, file, line) do { \ uintptr_t _tid = (uintptr_t)(tid); \ - uintptr_t _v; \ + uintptr_t _v = RW_UNLOCKED; \ \ - if (!_rw_write_lock_fetch((rw), &_v, _tid)) \ + if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__acquire) || \ + !_rw_write_lock_fetch((rw), &_v, _tid))) \ _rw_wlock_hard((rw), _v, _tid, (file), (line)); \ - else \ - LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, \ - 0, 0, file, line, LOCKSTAT_WRITER); \ } while (0) /* Release a write lock. */ @@ -118,9 +114,8 @@ if ((rw)->rw_recurse) \ (rw)->rw_recurse--; \ else { \ - LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, \ - LOCKSTAT_WRITER); \ - if (!_rw_write_unlock((rw), _tid)) \ + if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__release) ||\ + !_rw_write_unlock((rw), _tid))) \ _rw_wunlock_hard((rw), _tid, (file), (line)); \ } \ } while (0)