Date: Mon, 7 May 2018 23:36:16 +0000 (UTC) From: Mateusz Guzik <mjg@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r333344 - in head/sys: kern sys Message-ID: <201805072336.w47NaGXH001864@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: mjg Date: Mon May 7 23:36:16 2018 New Revision: 333344 URL: https://svnweb.freebsd.org/changeset/base/333344 Log: Inlined sched_userret. The tested condition is rarely true and it induces a function call on each return to userspace. Bumps getuid rate by about 1% on Broadwell. Modified: head/sys/kern/sched_4bsd.c head/sys/kern/sched_ule.c head/sys/sys/sched.h Modified: head/sys/kern/sched_4bsd.c ============================================================================== --- head/sys/kern/sched_4bsd.c Mon May 7 23:23:11 2018 (r333343) +++ head/sys/kern/sched_4bsd.c Mon May 7 23:36:16 2018 (r333344) @@ -1481,25 +1481,13 @@ sched_preempt(struct thread *td) } void -sched_userret(struct thread *td) +sched_userret_slowpath(struct thread *td) { - /* - * XXX we cheat slightly on the locking here to avoid locking in - * the usual case. Setting td_priority here is essentially an - * incomplete workaround for not setting it properly elsewhere. - * Now that some interrupt handlers are threads, not setting it - * properly elsewhere can clobber it in the window between setting - * it here and returning to user mode, so don't waste time setting - * it perfectly here. - */ - KASSERT((td->td_flags & TDF_BORROWING) == 0, - ("thread with borrowed priority returning to userland")); - if (td->td_priority != td->td_user_pri) { - thread_lock(td); - td->td_priority = td->td_user_pri; - td->td_base_pri = td->td_user_pri; - thread_unlock(td); - } + + thread_lock(td); + td->td_priority = td->td_user_pri; + td->td_base_pri = td->td_user_pri; + thread_unlock(td); } void Modified: head/sys/kern/sched_ule.c ============================================================================== --- head/sys/kern/sched_ule.c Mon May 7 23:23:11 2018 (r333343) +++ head/sys/kern/sched_ule.c Mon May 7 23:36:16 2018 (r333344) @@ -2356,26 +2356,14 @@ sched_preempt(struct thread *td) * to static priorities in msleep() or similar. */ void -sched_userret(struct thread *td) +sched_userret_slowpath(struct thread *td) { - /* - * XXX we cheat slightly on the locking here to avoid locking in - * the usual case. Setting td_priority here is essentially an - * incomplete workaround for not setting it properly elsewhere. - * Now that some interrupt handlers are threads, not setting it - * properly elsewhere can clobber it in the window between setting - * it here and returning to user mode, so don't waste time setting - * it perfectly here. - */ - KASSERT((td->td_flags & TDF_BORROWING) == 0, - ("thread with borrowed priority returning to userland")); - if (td->td_priority != td->td_user_pri) { - thread_lock(td); - td->td_priority = td->td_user_pri; - td->td_base_pri = td->td_user_pri; - tdq_setlowpri(TDQ_SELF(), td); - thread_unlock(td); - } + + thread_lock(td); + td->td_priority = td->td_user_pri; + td->td_base_pri = td->td_user_pri; + tdq_setlowpri(TDQ_SELF(), td); + thread_unlock(td); } /* Modified: head/sys/sys/sched.h ============================================================================== --- head/sys/sys/sched.h Mon May 7 23:23:11 2018 (r333343) +++ head/sys/sys/sched.h Mon May 7 23:36:16 2018 (r333344) @@ -103,13 +103,32 @@ void sched_switch(struct thread *td, struct thread *ne void sched_throw(struct thread *td); void sched_unlend_prio(struct thread *td, u_char prio); void sched_user_prio(struct thread *td, u_char prio); -void sched_userret(struct thread *td); +void sched_userret_slowpath(struct thread *td); void sched_wakeup(struct thread *td); #ifdef RACCT #ifdef SCHED_4BSD fixpt_t sched_pctcpu_delta(struct thread *td); #endif #endif + +static inline void +sched_userret(struct thread *td) +{ + + /* + * XXX we cheat slightly on the locking here to avoid locking in + * the usual case. Setting td_priority here is essentially an + * incomplete workaround for not setting it properly elsewhere. + * Now that some interrupt handlers are threads, not setting it + * properly elsewhere can clobber it in the window between setting + * it here and returning to user mode, so don't waste time setting + * it perfectly here. + */ + KASSERT((td->td_flags & TDF_BORROWING) == 0, + ("thread with borrowed priority returning to userland")); + if (__predict_false(td->td_priority != td->td_user_pri)) + sched_userret_slowpath(td); +} /* * Threads are moved on and off of run queues
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201805072336.w47NaGXH001864>