Date: Fri, 17 Sep 2021 01:40:16 GMT From: Alexander Motin <mav@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org Subject: git: b4194a432b59 - stable/13 - callout(9): Allow spin locks use with callout_init_mtx(). Message-ID: <202109170140.18H1eGFG064552@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch stable/13 has been updated by mav: URL: https://cgit.FreeBSD.org/src/commit/?id=b4194a432b599c9e96ce8cd7418e15447d4a4cdb commit b4194a432b599c9e96ce8cd7418e15447d4a4cdb Author: Alexander Motin <mav@FreeBSD.org> AuthorDate: 2021-09-03 01:16:46 +0000 Commit: Alexander Motin <mav@FreeBSD.org> CommitDate: 2021-09-17 01:40:15 +0000 callout(9): Allow spin locks use with callout_init_mtx(). Implement lock_spin()/unlock_spin() lock class methods, moving the assertion to _sleep() instead. Change assertions in callout(9) to allow spin locks for both regular and C_DIRECT_EXEC cases. In case of C_DIRECT_EXEC callouts spin locks are the only locks allowed actually. As the first use case allow taskqueue_enqueue_timeout() use on fast task queues. It actually becomes more efficient due to avoided extra context switches in callout(9) thanks to C_DIRECT_EXEC. MFC after: 2 weeks Reviewed by: hselasky Differential Revision: https://reviews.freebsd.org/D31778 (cherry picked from commit 4730a8972b1f4b67bf9ffde8e63ca906ef4c9563) --- share/man/man9/callout.9 | 5 +---- share/man/man9/taskqueue.9 | 5 +---- sys/kern/kern_mutex.c | 8 ++++++-- sys/kern/kern_synch.c | 2 ++ sys/kern/kern_timeout.c | 10 +++++----- sys/kern/subr_taskqueue.c | 3 ++- 6 files changed, 17 insertions(+), 16 deletions(-) diff --git a/share/man/man9/callout.9 b/share/man/man9/callout.9 index 9e0cf5e6dc5e..5c011a20d338 100644 --- a/share/man/man9/callout.9 +++ b/share/man/man9/callout.9 @@ -29,7 +29,7 @@ .\" .\" $FreeBSD$ .\" -.Dd December 13, 2019 +.Dd September 1, 2021 .Dt CALLOUT 9 .Os .Sh NAME @@ -241,9 +241,6 @@ and the associated lock is released. This ensures that stopping or rescheduling the callout will abort any previously scheduled invocation. .Pp -Only regular mutexes may be used with -.Fn callout_init_mtx ; -spin mutexes are not supported. A sleepable read-mostly lock .Po one initialized with the diff --git a/share/man/man9/taskqueue.9 b/share/man/man9/taskqueue.9 index 2ede90458629..58eb6b7c5571 100644 --- a/share/man/man9/taskqueue.9 +++ b/share/man/man9/taskqueue.9 @@ -28,7 +28,7 @@ .\" .\" $FreeBSD$ .\" -.Dd June 6, 2020 +.Dd September 1, 2021 .Dt TASKQUEUE 9 .Os .Sh NAME @@ -237,9 +237,6 @@ and .Va flags , as detailed in .Xr callout 9 . -Only non-fast task queues can be used for -.Va timeout_task -scheduling. If the .Va ticks argument is negative, the already scheduled enqueueing is not re-scheduled. diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c index d9db69e2ac09..54a96603ece3 100644 --- a/sys/kern/kern_mutex.c +++ b/sys/kern/kern_mutex.c @@ -215,7 +215,7 @@ void lock_spin(struct lock_object *lock, uintptr_t how) { - panic("spin locks can only use msleep_spin"); + mtx_lock_spin((struct mtx *)lock); } uintptr_t @@ -232,8 +232,12 @@ unlock_mtx(struct lock_object *lock) uintptr_t unlock_spin(struct lock_object *lock) { + struct mtx *m; - panic("spin locks can only use msleep_spin"); + m = (struct mtx *)lock; + mtx_assert(m, MA_OWNED | MA_NOTRECURSED); + mtx_unlock_spin(m); + return (0); } #ifdef KDTRACE_HOOKS diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index b63877e26b68..3d51af2740fc 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -187,6 +187,8 @@ _sleep(const void *ident, struct lock_object *lock, int priority, DROP_GIANT(); if (lock != NULL && lock != &Giant.lock_object && !(class->lc_flags & LC_SLEEPABLE)) { + KASSERT(!(class->lc_flags & LC_SPINLOCK), + ("spin locks can only use msleep_spin")); WITNESS_SAVE(lock, lock_witness); lock_state = class->lc_unlock(lock); } else diff --git a/sys/kern/kern_timeout.c b/sys/kern/kern_timeout.c index d9cf2784f642..992a093d30ab 100644 --- a/sys/kern/kern_timeout.c +++ b/sys/kern/kern_timeout.c @@ -919,8 +919,9 @@ callout_reset_sbt_on(struct callout *c, sbintime_t sbt, sbintime_t prec, } else { direct = 0; } - KASSERT(!direct || c->c_lock == NULL, - ("%s: direct callout %p has lock", __func__, c)); + KASSERT(!direct || c->c_lock == NULL || + (LOCK_CLASS(c->c_lock)->lc_flags & LC_SPINLOCK), + ("%s: direct callout %p has non-spin lock", __func__, c)); cc = callout_lock(c); /* * Don't allow migration if the user does not care. @@ -1332,9 +1333,8 @@ _callout_init_lock(struct callout *c, struct lock_object *lock, int flags) ("callout_init_lock: bad flags %d", flags)); KASSERT(lock != NULL || (flags & CALLOUT_RETURNUNLOCKED) == 0, ("callout_init_lock: CALLOUT_RETURNUNLOCKED with no lock")); - KASSERT(lock == NULL || !(LOCK_CLASS(lock)->lc_flags & - (LC_SPINLOCK | LC_SLEEPABLE)), ("%s: invalid lock class", - __func__)); + KASSERT(lock == NULL || !(LOCK_CLASS(lock)->lc_flags & LC_SLEEPABLE), + ("%s: callout %p has sleepable lock", __func__, c)); c->c_iflags = flags & (CALLOUT_RETURNUNLOCKED | CALLOUT_SHAREDLOCK); c->c_cpu = cc_default_cpu; } diff --git a/sys/kern/subr_taskqueue.c b/sys/kern/subr_taskqueue.c index 061361cc06d7..e43b09010761 100644 --- a/sys/kern/subr_taskqueue.c +++ b/sys/kern/subr_taskqueue.c @@ -309,7 +309,6 @@ taskqueue_enqueue_timeout_sbt(struct taskqueue *queue, TQ_LOCK(queue); KASSERT(timeout_task->q == NULL || timeout_task->q == queue, ("Migrated queue")); - KASSERT(!queue->tq_spin, ("Timeout for spin-queue")); timeout_task->q = queue; res = timeout_task->t.ta_pending; if (timeout_task->f & DT_DRAIN_IN_PROGRESS) { @@ -329,6 +328,8 @@ taskqueue_enqueue_timeout_sbt(struct taskqueue *queue, sbt = -sbt; /* Ignore overflow. */ } if (sbt > 0) { + if (queue->tq_spin) + flags |= C_DIRECT_EXEC; callout_reset_sbt(&timeout_task->c, sbt, pr, taskqueue_timeout_func, timeout_task, flags); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202109170140.18H1eGFG064552>