From owner-svn-src-all@freebsd.org Tue Feb 7 17:04:32 2017 Return-Path: Delivered-To: svn-src-all@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 6228BCD513C; Tue, 7 Feb 2017 17:04:32 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 3CDA6CA; Tue, 7 Feb 2017 17:04:32 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v17H4VUx072064; Tue, 7 Feb 2017 17:04:31 GMT (envelope-from mjg@FreeBSD.org) Received: (from mjg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v17H4VMH072062; Tue, 7 Feb 2017 17:04:31 GMT (envelope-from mjg@FreeBSD.org) Message-Id: <201702071704.v17H4VMH072062@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mjg set sender to mjg@FreeBSD.org using -f From: Mateusz Guzik Date: Tue, 7 Feb 2017 17:04:31 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r313392 - in head/sys: kern sys X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 07 Feb 2017 17:04:32 -0000 Author: mjg Date: Tue Feb 7 17:04:31 2017 New Revision: 313392 URL: https://svnweb.freebsd.org/changeset/base/313392 Log: rwlock: implement RW_LOCK_WRITER_RECURSED bit This moves recursion handling out of the inlined wunlock path and in particular saves a read and a branch. Discussed with: Modified: head/sys/kern/kern_rwlock.c head/sys/sys/rwlock.h Modified: head/sys/kern/kern_rwlock.c ============================================================================== --- head/sys/kern/kern_rwlock.c Tue Feb 7 17:03:22 2017 (r313391) +++ head/sys/kern/kern_rwlock.c Tue Feb 7 17:04:31 2017 (r313392) @@ -312,6 +312,7 @@ __rw_try_wlock(volatile uintptr_t *c, co if (rw_wlocked(rw) && (rw->lock_object.lo_flags & LO_RECURSABLE) != 0) { rw->rw_recurse++; + atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED); rval = 1; } else rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED, @@ -345,10 +346,8 @@ _rw_wunlock_cookie(volatile uintptr_t *c WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line); - if (rw->rw_recurse) - rw->rw_recurse--; - else - _rw_wunlock_hard(rw, (uintptr_t)curthread, file, line); + + _rw_wunlock_hard(rw, (uintptr_t)curthread, file, line); TD_LOCKS_DEC(curthread); } @@ -802,6 +801,7 @@ __rw_wlock_hard(volatile uintptr_t *c, u ("%s: recursing but non-recursive rw %s @ %s:%d\n", __func__, rw->lock_object.lo_name, file, line)); rw->rw_recurse++; + atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED); if (LOCK_LOG_TEST(&rw->lock_object, 0)) CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw); return; @@ -994,12 +994,17 @@ __rw_wunlock_hard(volatile uintptr_t *c, return; rw = rwlock2rw(c); - MPASS(!rw_recursed(rw)); - LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, - LOCKSTAT_WRITER); - if (_rw_write_unlock(rw, tid)) + if (!rw_recursed(rw)) { + LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, + LOCKSTAT_WRITER); + if (_rw_write_unlock(rw, tid)) + return; + } else { + if (--(rw->rw_recurse) == 0) + atomic_clear_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED); return; + } KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), ("%s: neither of the waiter flags are set", __func__)); Modified: head/sys/sys/rwlock.h ============================================================================== --- head/sys/sys/rwlock.h Tue Feb 7 17:03:22 2017 (r313391) +++ head/sys/sys/rwlock.h Tue Feb 7 17:04:31 2017 (r313392) @@ -58,9 +58,10 @@ #define RW_LOCK_READ_WAITERS 0x02 #define RW_LOCK_WRITE_WAITERS 0x04 #define RW_LOCK_WRITE_SPINNER 0x08 +#define RW_LOCK_WRITER_RECURSED 0x10 #define RW_LOCK_FLAGMASK \ (RW_LOCK_READ | RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS | \ - RW_LOCK_WRITE_SPINNER) + RW_LOCK_WRITE_SPINNER | RW_LOCK_WRITER_RECURSED) #define RW_LOCK_WAITERS (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS) #define RW_OWNER(x) ((x) & ~RW_LOCK_FLAGMASK) @@ -111,13 +112,9 @@ #define __rw_wunlock(rw, tid, file, line) do { \ uintptr_t _tid = (uintptr_t)(tid); \ \ - if ((rw)->rw_recurse) \ - (rw)->rw_recurse--; \ - else { \ - if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__release) ||\ - !_rw_write_unlock((rw), _tid))) \ - _rw_wunlock_hard((rw), _tid, (file), (line)); \ - } \ + if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__release) || \ + !_rw_write_unlock((rw), _tid))) \ + _rw_wunlock_hard((rw), _tid, (file), (line)); \ } while (0) /*