From owner-svn-src-user@FreeBSD.ORG Tue Apr 24 12:15:51 2012 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 4A4D7106566C; Tue, 24 Apr 2012 12:15:51 +0000 (UTC) (envelope-from andre@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 34D278FC08; Tue, 24 Apr 2012 12:15:51 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.4/8.14.4) with ESMTP id q3OCFprA048139; Tue, 24 Apr 2012 12:15:51 GMT (envelope-from andre@svn.freebsd.org) Received: (from andre@localhost) by svn.freebsd.org (8.14.4/8.14.4/Submit) id q3OCFoYf048135; Tue, 24 Apr 2012 12:15:50 GMT (envelope-from andre@svn.freebsd.org) Message-Id: <201204241215.q3OCFoYf048135@svn.freebsd.org> From: Andre Oppermann Date: Tue, 24 Apr 2012 12:15:50 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r234648 - in user/andre/routelocking: kern sys X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 24 Apr 2012 12:15:51 -0000 Author: andre Date: Tue Apr 24 12:15:50 2012 New Revision: 234648 URL: http://svn.freebsd.org/changeset/base/234648 Log: Add INVARIANT and WITNESS support to rm_lock locks and optimize the synchronization path by replacing a LIST of active readers with a TAILQ. Obtained from: Isilon Submitted by: mlaier Modified: user/andre/routelocking/kern/kern_rmlock.c user/andre/routelocking/sys/_rmlock.h user/andre/routelocking/sys/rmlock.h Modified: user/andre/routelocking/kern/kern_rmlock.c ============================================================================== --- user/andre/routelocking/kern/kern_rmlock.c Tue Apr 24 11:46:15 2012 (r234647) +++ user/andre/routelocking/kern/kern_rmlock.c Tue Apr 24 12:15:50 2012 (r234648) @@ -102,15 +102,25 @@ assert_rm(const struct lock_object *lock static void lock_rm(struct lock_object *lock, int how) { + struct rmlock *rm; - panic("lock_rm called"); + rm = (struct rmlock *)lock; + if (how) + rm_wlock(rm); + else + panic("lock_rm called in shared mode"); } static int unlock_rm(struct lock_object *lock) { + struct rmlock *rm; - panic("unlock_rm called"); + rm = (struct rmlock *)lock; + if (!rm_wowned(rm)) + panic("unlock_rm called without exclusive lock held"); + rm_wunlock(rm); + return (1); } #ifdef KDTRACE_HOOKS @@ -167,6 +177,7 @@ rm_tracker_remove(struct pcpu *pc, struc static void rm_cleanIPI(void *arg) { + TAILQ_HEAD(,rm_priotracker) tmp_list = TAILQ_HEAD_INITIALIZER(tmp_list); struct pcpu *pc; struct rmlock *rm = arg; struct rm_priotracker *tracker; @@ -178,12 +189,12 @@ rm_cleanIPI(void *arg) tracker = (struct rm_priotracker *)queue; if (tracker->rmp_rmlock == rm && tracker->rmp_flags == 0) { tracker->rmp_flags = RMPF_ONQUEUE; - mtx_lock_spin(&rm_spinlock); - LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker, - rmp_qentry); - mtx_unlock_spin(&rm_spinlock); + TAILQ_INSERT_HEAD(&tmp_list, tracker, rmp_qentry); } } + mtx_lock_spin(&rm_spinlock); + TAILQ_CONCAT(&rm->rm_activeReaders, &tmp_list, rmp_qentry); + mtx_unlock_spin(&rm_spinlock); } CTASSERT((RM_SLEEPABLE & LO_CLASSFLAGS) == RM_SLEEPABLE); @@ -199,7 +210,7 @@ rm_init_flags(struct rmlock *rm, const c if (opts & RM_RECURSE) liflags |= LO_RECURSABLE; rm->rm_writecpus = all_cpus; - LIST_INIT(&rm->rm_activeReaders); + TAILQ_INIT(&rm->rm_activeReaders); if (opts & RM_SLEEPABLE) { liflags |= RM_SLEEPABLE; sx_init_flags(&rm->rm_lock_sx, "rmlock_sx", SX_RECURSE); @@ -227,7 +238,7 @@ rm_destroy(struct rmlock *rm) } int -rm_wowned(const struct rmlock *rm) +rm_wowned(struct rmlock *rm) { if (rm->lock_object.lo_flags & RM_SLEEPABLE) @@ -294,7 +305,7 @@ _rm_rlock_hard(struct rmlock *rm, struct if ((atracker->rmp_rmlock == rm) && (atracker->rmp_thread == tracker->rmp_thread)) { mtx_lock_spin(&rm_spinlock); - LIST_INSERT_HEAD(&rm->rm_activeReaders, + TAILQ_INSERT_HEAD(&rm->rm_activeReaders, tracker, rmp_qentry); tracker->rmp_flags = RMPF_ONQUEUE; mtx_unlock_spin(&rm_spinlock); @@ -390,7 +401,8 @@ _rm_unlock_hard(struct thread *td,struct return; mtx_lock_spin(&rm_spinlock); - LIST_REMOVE(tracker, rmp_qentry); + TAILQ_REMOVE(&tracker->rmp_rmlock->rm_activeReaders, tracker, + rmp_qentry); if (tracker->rmp_flags & RMPF_SIGNAL) { struct rmlock *rm; @@ -468,7 +480,7 @@ _rm_wlock(struct rmlock *rm) #endif mtx_lock_spin(&rm_spinlock); - while ((prio = LIST_FIRST(&rm->rm_activeReaders)) != NULL) { + while ((prio = TAILQ_FIRST(&rm->rm_activeReaders)) != NULL) { ts = turnstile_trywait(&rm->lock_object); prio->rmp_flags = RMPF_ONQUEUE | RMPF_SIGNAL; mtx_unlock_spin(&rm_spinlock); @@ -609,3 +621,92 @@ _rm_runlock_debug(struct rmlock *rm, str } #endif + +#ifdef INVARIANT_SUPPORT +void +_rm_assert(struct rmlock *rm, int what, const char *file, int line) +{ +#ifndef WITNESS + struct pcpu *pc; + struct rm_queue *queue; + struct rm_priotracker *atracker; + int cnt; +#endif + + if (panicstr != NULL) + return; + switch (what) { + case RM_LOCKED: + case RM_LOCKED | RM_RECURSED: + case RM_LOCKED | RM_NOTRECURSED: + case RM_RLOCKED | RM_RECURSED: + case RM_RLOCKED | RM_NOTRECURSED: + case RM_RLOCKED: +#ifdef WITNESS + witness_assert(&rm->lock_object, what, file, line); +#else + if ((what == RM_RLOCKED) && rm_wowned(rm)) + panic("Lock %s writelocked @ %s:%d\n", + rm->lock_object.lo_name, file, line); + + critical_enter(); + pc = pcpu_find(curcpu); + cnt = 0; + for (queue = pc->pc_rm_queue.rmq_next; + queue != &pc->pc_rm_queue; queue = queue->rmq_next) { + atracker = (struct rm_priotracker *)queue; + if ((atracker->rmp_rmlock == rm) && + (atracker->rmp_thread == curthread)) + cnt++; + } + critical_exit(); + + if ((cnt == 0) && !rm_wowned(rm)) + panic("Lock %s not %slocked @ %s:%d\n", + rm->lock_object.lo_name, (what == RM_RLOCKED) ? + "read " : "", file, line); + if (cnt > 2) { + if (what & RA_NOTRECURSED) + panic("Lock %s recursed @ %s:%d\n", + rm->lock_object.lo_name, file, line); + } else if (what & RA_RECURSED) + panic("Lock %s not recursed @ %s:%d\n", + rm->lock_object.lo_name, file, line); +#endif + break; + case RM_WLOCKED: + if (!rm_wowned(rm)) + panic("Lock %s not writelocked @ %s:%d\n", + rm->lock_object.lo_name, file, line); + break; + case RM_UNLOCKED: +#ifdef WITNESS + witness_assert(&rm->lock_object, what, file, line); +#else + if (rm_wowned(rm)) + panic("Lock %s writelocked @ %s:%d\n", + rm->lock_object.lo_name, file, line); + + critical_enter(); + pc = pcpu_find(curcpu); + cnt = 0; + for (queue = pc->pc_rm_queue.rmq_next; + queue != &pc->pc_rm_queue; queue = queue->rmq_next) { + atracker = (struct rm_priotracker *)queue; + if ((atracker->rmp_rmlock == rm) && + (atracker->rmp_thread == curthread)) + cnt++; + } + critical_exit(); + + if (cnt != 0) + panic("Lock %s readlocked @ %s:%d\n", + rm->lock_object.lo_name, file, line); +#endif + break; + default: + panic("Unknown rm lock assertion: %d @ %s:%d", what, file, + line); + } +} +#endif /* INVARIANT_SUPPORT */ Modified: user/andre/routelocking/sys/_rmlock.h ============================================================================== --- user/andre/routelocking/sys/_rmlock.h Tue Apr 24 11:46:15 2012 (r234647) +++ user/andre/routelocking/sys/_rmlock.h Tue Apr 24 12:15:50 2012 (r234648) @@ -35,18 +35,18 @@ /* * XXXUPS remove as soon as we have per cpu variable * linker sets and can define rm_queue in _rm_lock.h -*/ + */ #include /* * Mostly reader/occasional writer lock. */ -LIST_HEAD(rmpriolist,rm_priotracker); +TAILQ_HEAD(rmpriolist, rm_priotracker); struct rmlock { struct lock_object lock_object; volatile cpuset_t rm_writecpus; - LIST_HEAD(,rm_priotracker) rm_activeReaders; + TAILQ_HEAD(, rm_priotracker) rm_activeReaders; union { struct mtx _rm_lock_mtx; struct sx _rm_lock_sx; @@ -60,7 +60,7 @@ struct rm_priotracker { struct rmlock *rmp_rmlock; struct thread *rmp_thread; int rmp_flags; - LIST_ENTRY(rm_priotracker) rmp_qentry; + TAILQ_ENTRY(rm_priotracker) rmp_qentry; }; #endif /* !_SYS__RMLOCK_H_ */ Modified: user/andre/routelocking/sys/rmlock.h ============================================================================== --- user/andre/routelocking/sys/rmlock.h Tue Apr 24 11:46:15 2012 (r234647) +++ user/andre/routelocking/sys/rmlock.h Tue Apr 24 12:15:50 2012 (r234648) @@ -49,7 +49,7 @@ void rm_init(struct rmlock *rm, const char *name); void rm_init_flags(struct rmlock *rm, const char *name, int opts); void rm_destroy(struct rmlock *rm); -int rm_wowned(const struct rmlock *rm); +int rm_wowned(struct rmlock *rm); void rm_sysinit(void *arg); void rm_sysinit_flags(void *arg); @@ -65,6 +65,9 @@ void _rm_wunlock(struct rmlock *rm); int _rm_rlock(struct rmlock *rm, struct rm_priotracker *tracker, int trylock); void _rm_runlock(struct rmlock *rm, struct rm_priotracker *tracker); +#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) +void _rm_assert(struct rmlock *rm, int what, const char *file, int line); +#endif /* * Public interface for lock operations. @@ -89,6 +92,8 @@ void _rm_runlock(struct rmlock *rm, str #define rm_try_rlock(rm,tracker) _rm_rlock((rm),(tracker), 1) #define rm_runlock(rm,tracker) _rm_runlock((rm), (tracker)) #endif +#define rm_sleep(chan, rm, pri, wmesg, timo) \ + _sleep((chan), &(rm)->lock_object, (pri), (wmesg), (timo)) struct rm_args { struct rmlock *ra_rm; @@ -123,5 +128,20 @@ struct rm_args_flags { SYSUNINIT(name##_rm_sysuninit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \ rm_destroy, (rm)) +#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) +#define RM_LOCKED LA_LOCKED +#define RM_RLOCKED LA_SLOCKED +#define RM_WLOCKED LA_XLOCKED +#define RM_UNLOCKED LA_UNLOCKED +#define RM_RECURSED LA_RECURSED +#define RM_NOTRECURSED LA_NOTRECURSED +#endif + +#ifdef INVARIANTS +#define rm_assert(rm, what) _rm_assert((rm), (what), LOCK_FILE, LOCK_LINE) +#else +#define rm_assert(rm, what) +#endif + #endif /* _KERNEL */ #endif /* !_SYS_RMLOCK_H_ */