Date: Sat, 21 Oct 2017 22:40:09 +0000 (UTC) From: Mateusz Guzik <mjg@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r324836 - in head/sys: kern sys Message-ID: <201710212240.v9LMe9iK096074@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: mjg Date: Sat Oct 21 22:40:09 2017 New Revision: 324836 URL: https://svnweb.freebsd.org/changeset/base/324836 Log: mtx: implement thread lock fastpath MFC after: 1 week Modified: head/sys/kern/kern_mutex.c head/sys/sys/mutex.h Modified: head/sys/kern/kern_mutex.c ============================================================================== --- head/sys/kern/kern_mutex.c Sat Oct 21 21:58:24 2017 (r324835) +++ head/sys/kern/kern_mutex.c Sat Oct 21 22:40:09 2017 (r324836) @@ -791,7 +791,67 @@ _mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t } #endif /* SMP */ +#ifdef INVARIANTS +static void +thread_lock_validate(struct mtx *m, int opts, const char *file, int line) +{ + + KASSERT(m->mtx_lock != MTX_DESTROYED, + ("thread_lock() of destroyed mutex @ %s:%d", file, line)); + KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, + ("thread_lock() of sleep mutex %s @ %s:%d", + m->lock_object.lo_name, file, line)); + if (mtx_owned(m)) + KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, + ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n", + m->lock_object.lo_name, file, line)); + WITNESS_CHECKORDER(&m->lock_object, + opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); +} +#else +#define thread_lock_validate(m, opts, file, line) do { } while (0) +#endif + +#ifndef LOCK_PROFILING +#if LOCK_DEBUG > 0 void +_thread_lock(struct thread *td, int opts, const char *file, int line) +#else +void +_thread_lock(struct thread *td) +#endif +{ + struct mtx *m; + uintptr_t tid, v; + + tid = (uintptr_t)curthread; + + spinlock_enter(); + m = td->td_lock; + thread_lock_validate(m, 0, file, line); + v = MTX_READ_VALUE(m); + if (__predict_true(v == MTX_UNOWNED)) { + if (__predict_false(!_mtx_obtain_lock(m, tid))) + goto slowpath_unlocked; + } else if (v == tid) { + m->mtx_recurse++; + } else + goto slowpath_unlocked; + if (__predict_true(m == td->td_lock)) { + WITNESS_LOCK(&m->lock_object, LOP_EXCLUSIVE, file, line); + return; + } + if (m->mtx_recurse != 0) + m->mtx_recurse--; + else + _mtx_release_lock_quick(m); +slowpath_unlocked: + spinlock_exit(); + thread_lock_flags_(td, 0, 0, 0); +} +#endif + +void thread_lock_flags_(struct thread *td, int opts, const char *file, int line) { struct mtx *m; @@ -834,17 +894,7 @@ retry: v = MTX_UNOWNED; spinlock_enter(); m = td->td_lock; - KASSERT(m->mtx_lock != MTX_DESTROYED, - ("thread_lock() of destroyed mutex @ %s:%d", file, line)); - KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, - ("thread_lock() of sleep mutex %s @ %s:%d", - m->lock_object.lo_name, file, line)); - if (mtx_owned(m)) - KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, - ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n", - m->lock_object.lo_name, file, line)); - WITNESS_CHECKORDER(&m->lock_object, - opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); + thread_lock_validate(m, opts, file, line); for (;;) { if (_mtx_obtain_lock_fetch(m, &v, tid)) break; Modified: head/sys/sys/mutex.h ============================================================================== --- head/sys/sys/mutex.h Sat Oct 21 21:58:24 2017 (r324835) +++ head/sys/sys/mutex.h Sat Oct 21 22:40:09 2017 (r324836) @@ -127,9 +127,23 @@ void __mtx_assert(const volatile uintptr_t *c, int wha int line); #endif void thread_lock_flags_(struct thread *, int, const char *, int); +#if LOCK_DEBUG > 0 +void _thread_lock(struct thread *td, int opts, const char *file, int line); +#else +void _thread_lock(struct thread *); +#endif +#if defined(LOCK_PROFILING) || defined(KLD_MODULE) #define thread_lock(tdp) \ thread_lock_flags_((tdp), 0, __FILE__, __LINE__) +#elif LOCK_DEBUG > 0 +#define thread_lock(tdp) \ + _thread_lock((tdp), 0, __FILE__, __LINE__) +#else +#define thread_lock(tdp) \ + _thread_lock((tdp)) +#endif + #define thread_lock_flags(tdp, opt) \ thread_lock_flags_((tdp), (opt), __FILE__, __LINE__) #define thread_unlock(tdp) \
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201710212240.v9LMe9iK096074>