From owner-p4-projects@FreeBSD.ORG Thu Dec 30 20:59:53 2004 Return-Path: Delivered-To: p4-projects@freebsd.org Received: by hub.freebsd.org (Postfix, from userid 32767) id 3D0EA16A4D0; Thu, 30 Dec 2004 20:59:53 +0000 (GMT) Delivered-To: perforce@freebsd.org Received: from mx1.FreeBSD.org (mx1.freebsd.org [216.136.204.125]) by hub.freebsd.org (Postfix) with ESMTP id 1143D16A4CE for ; Thu, 30 Dec 2004 20:59:53 +0000 (GMT) Received: from repoman.freebsd.org (repoman.freebsd.org [216.136.204.115]) by mx1.FreeBSD.org (Postfix) with ESMTP id A511E43D31 for ; Thu, 30 Dec 2004 20:59:52 +0000 (GMT) (envelope-from jhb@freebsd.org) Received: from repoman.freebsd.org (localhost [127.0.0.1]) by repoman.freebsd.org (8.13.1/8.13.1) with ESMTP id iBUKxq7J045299 for ; Thu, 30 Dec 2004 20:59:52 GMT (envelope-from jhb@freebsd.org) Received: (from perforce@localhost) by repoman.freebsd.org (8.13.1/8.13.1/Submit) id iBUKxqHh045296 for perforce@freebsd.org; Thu, 30 Dec 2004 20:59:52 GMT (envelope-from jhb@freebsd.org) Date: Thu, 30 Dec 2004 20:59:52 GMT Message-Id: <200412302059.iBUKxqHh045296@repoman.freebsd.org> X-Authentication-Warning: repoman.freebsd.org: perforce set sender to jhb@freebsd.org using -f From: John Baldwin To: Perforce Change Reviews Subject: PERFORCE change 67928 for review X-BeenThere: p4-projects@freebsd.org X-Mailman-Version: 2.1.1 Precedence: list List-Id: p4 projects tree changes List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 30 Dec 2004 20:59:53 -0000 http://perforce.freebsd.org/chv.cgi?CH=67928 Change 67928 by jhb@jhb_slimer on 2004/12/30 20:59:50 IFC @67927. Loop back priority inversion fixes. Affected files ... .. //depot/projects/smpng/sys/compat/ndis/subr_hal.c#9 integrate .. //depot/projects/smpng/sys/dev/md/md.c#61 integrate .. //depot/projects/smpng/sys/geom/geom_kern.c#19 integrate .. //depot/projects/smpng/sys/geom/mirror/g_mirror.c#15 integrate .. //depot/projects/smpng/sys/geom/raid3/g_raid3.c#12 integrate .. //depot/projects/smpng/sys/kern/kern_intr.c#64 integrate .. //depot/projects/smpng/sys/kern/kern_resource.c#53 integrate .. //depot/projects/smpng/sys/kern/kern_synch.c#88 integrate .. //depot/projects/smpng/sys/kern/sched_4bsd.c#45 integrate .. //depot/projects/smpng/sys/kern/sched_ule.c#49 integrate .. //depot/projects/smpng/sys/kern/subr_trap.c#75 integrate .. //depot/projects/smpng/sys/kern/subr_turnstile.c#18 integrate .. //depot/projects/smpng/sys/sys/proc.h#137 integrate .. //depot/projects/smpng/sys/sys/sched.h#18 integrate .. //depot/projects/smpng/sys/sys/turnstile.h#5 integrate Differences ... ==== //depot/projects/smpng/sys/compat/ndis/subr_hal.c#9 (text+ko) ==== @@ -31,7 +31,7 @@ */ #include -__FBSDID("$FreeBSD: src/sys/compat/ndis/subr_hal.c,v 1.13 2004/08/01 20:04:30 wpaul Exp $"); +__FBSDID("$FreeBSD: src/sys/compat/ndis/subr_hal.c,v 1.14 2004/12/30 20:29:58 jhb Exp $"); #include #include @@ -308,7 +308,6 @@ mtx_lock_spin(&sched_lock); oldirql = curthread->td_base_pri; sched_prio(curthread, PI_REALTIME); - curthread->td_base_pri = PI_REALTIME; mtx_unlock_spin(&sched_lock); return(oldirql); @@ -324,7 +323,6 @@ panic("IRQL_NOT_GREATER_THAN"); mtx_lock_spin(&sched_lock); - curthread->td_base_pri = oldirql; sched_prio(curthread, oldirql); mtx_unlock_spin(&sched_lock); ==== //depot/projects/smpng/sys/dev/md/md.c#61 (text+ko) ==== @@ -6,7 +6,7 @@ * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp * ---------------------------------------------------------------------------- * - * $FreeBSD: src/sys/dev/md/md.c,v 1.146 2004/12/27 17:20:06 pjd Exp $ + * $FreeBSD: src/sys/dev/md/md.c,v 1.147 2004/12/30 20:29:58 jhb Exp $ * */ @@ -71,6 +71,7 @@ #include #include #include +#include #include #include #include @@ -618,7 +619,9 @@ int error, hasgiant; sc = arg; - curthread->td_base_pri = PRIBIO; + mtx_lock_spin(&sched_lock); + sched_prio(curthread, PRIBIO); + mtx_unlock_spin(&sched_lock); switch (sc->type) { case MD_VNODE: ==== //depot/projects/smpng/sys/geom/geom_kern.c#19 (text+ko) ==== @@ -34,7 +34,7 @@ */ #include -__FBSDID("$FreeBSD: src/sys/geom/geom_kern.c,v 1.35 2004/09/13 14:58:27 pjd Exp $"); +__FBSDID("$FreeBSD: src/sys/geom/geom_kern.c,v 1.36 2004/12/30 20:29:58 jhb Exp $"); #include #include @@ -47,8 +47,9 @@ #include #include #include +#include +#include #include -#include #include #include @@ -87,7 +88,9 @@ struct thread *tp = FIRST_THREAD_IN_PROC(p); mtx_assert(&Giant, MA_NOTOWNED); - tp->td_base_pri = PRIBIO; + mtx_lock_spin(&sched_lock); + sched_prio(tp, PRIBIO); + mtx_unlock_spin(&sched_lock); for(;;) { g_io_schedule_up(tp); } @@ -108,7 +111,9 @@ struct thread *tp = FIRST_THREAD_IN_PROC(p); mtx_assert(&Giant, MA_NOTOWNED); - tp->td_base_pri = PRIBIO; + mtx_lock_spin(&sched_lock); + sched_prio(tp, PRIBIO); + mtx_unlock_spin(&sched_lock); for(;;) { g_io_schedule_down(tp); } @@ -129,7 +134,9 @@ struct thread *tp = FIRST_THREAD_IN_PROC(p); mtx_assert(&Giant, MA_NOTOWNED); - tp->td_base_pri = PRIBIO; + mtx_lock_spin(&sched_lock); + sched_prio(tp, PRIBIO); + mtx_unlock_spin(&sched_lock); for(;;) { g_run_events(); tsleep(&g_wait_event, PRIBIO, "-", hz/10); ==== //depot/projects/smpng/sys/geom/mirror/g_mirror.c#15 (text+ko) ==== @@ -25,7 +25,7 @@ */ #include -__FBSDID("$FreeBSD: src/sys/geom/mirror/g_mirror.c,v 1.54 2004/12/23 21:15:15 pjd Exp $"); +__FBSDID("$FreeBSD: src/sys/geom/mirror/g_mirror.c,v 1.55 2004/12/30 20:29:58 jhb Exp $"); #include #include @@ -42,6 +42,7 @@ #include #include #include +#include #include @@ -1478,7 +1479,9 @@ u_int nreqs; sc = arg; - curthread->td_base_pri = PRIBIO; + mtx_lock_spin(&sched_lock); + sched_prio(curthread, PRIBIO); + mtx_unlock_spin(&sched_lock); nreqs = 0; for (;;) { ==== //depot/projects/smpng/sys/geom/raid3/g_raid3.c#12 (text+ko) ==== @@ -25,7 +25,7 @@ */ #include -__FBSDID("$FreeBSD: src/sys/geom/raid3/g_raid3.c,v 1.33 2004/12/28 21:52:45 pjd Exp $"); +__FBSDID("$FreeBSD: src/sys/geom/raid3/g_raid3.c,v 1.34 2004/12/30 20:29:58 jhb Exp $"); #include #include @@ -43,6 +43,7 @@ #include #include #include +#include #include @@ -1729,7 +1730,9 @@ u_int nreqs; sc = arg; - curthread->td_base_pri = PRIBIO; + mtx_lock_spin(&sched_lock); + sched_prio(curthread, PRIBIO); + mtx_unlock_spin(&sched_lock); nreqs = 0; for (;;) { ==== //depot/projects/smpng/sys/kern/kern_intr.c#64 (text+ko) ==== @@ -25,7 +25,7 @@ */ #include -__FBSDID("$FreeBSD: src/sys/kern/kern_intr.c,v 1.120 2004/11/17 14:39:41 jhb Exp $"); +__FBSDID("$FreeBSD: src/sys/kern/kern_intr.c,v 1.121 2004/12/30 20:29:58 jhb Exp $"); #include "opt_ddb.h" @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -143,14 +144,12 @@ ih = TAILQ_FIRST(&ithd->it_handlers); if (ih == NULL) { mtx_lock_spin(&sched_lock); - td->td_priority = PRI_MAX_ITHD; - td->td_base_pri = PRI_MAX_ITHD; + sched_prio(td, PRI_MAX_ITHD); mtx_unlock_spin(&sched_lock); return; } mtx_lock_spin(&sched_lock); - td->td_priority = ih->ih_pri; - td->td_base_pri = ih->ih_pri; + sched_prio(td, ih->ih_pri); mtx_unlock_spin(&sched_lock); missed = 0; TAILQ_FOREACH(ih, &ithd->it_handlers, ih_next) { ==== //depot/projects/smpng/sys/kern/kern_resource.c#53 (text+ko) ==== @@ -35,7 +35,7 @@ */ #include -__FBSDID("$FreeBSD: src/sys/kern/kern_resource.c,v 1.146 2004/10/05 18:51:11 jhb Exp $"); +__FBSDID("$FreeBSD: src/sys/kern/kern_resource.c,v 1.147 2004/12/30 20:29:58 jhb Exp $"); #include "opt_compat.h" @@ -423,7 +423,6 @@ } sched_class(kg, rtp->type); if (curthread->td_ksegrp == kg) { - curthread->td_base_pri = kg->kg_user_pri; sched_prio(curthread, kg->kg_user_pri); /* XXX dubious */ } return (0); ==== //depot/projects/smpng/sys/kern/kern_synch.c#88 (text+ko) ==== @@ -35,7 +35,7 @@ */ #include -__FBSDID("$FreeBSD: src/sys/kern/kern_synch.c,v 1.266 2004/12/26 00:14:21 jeff Exp $"); +__FBSDID("$FreeBSD: src/sys/kern/kern_synch.c,v 1.267 2004/12/30 20:29:58 jhb Exp $"); #include "opt_ktrace.h" @@ -212,8 +212,6 @@ /* * Adjust this thread's priority. - * - * XXX: do we need to save priority in td_base_pri? */ mtx_lock_spin(&sched_lock); sched_prio(td, priority & PRIMASK); ==== //depot/projects/smpng/sys/kern/sched_4bsd.c#45 (text+ko) ==== @@ -33,7 +33,7 @@ */ #include -__FBSDID("$FreeBSD: src/sys/kern/sched_4bsd.c,v 1.70 2004/12/26 00:16:24 jeff Exp $"); +__FBSDID("$FreeBSD: src/sys/kern/sched_4bsd.c,v 1.71 2004/12/30 20:52:44 jhb Exp $"); #define kse td_sched @@ -50,6 +50,7 @@ #include #include #include +#include #include /* @@ -159,10 +160,12 @@ static void roundrobin(void *arg); static void schedcpu(void); static void schedcpu_thread(void); +static void sched_priority(struct thread *td, u_char prio); static void sched_setup(void *dummy); static void maybe_resched(struct thread *td); static void updatepri(struct ksegrp *kg); static void resetpriority(struct ksegrp *kg); +static void resetpriority_thread(struct thread *td, struct ksegrp *kg); #ifdef SMP static int forward_wakeup(int cpunum); #endif @@ -516,9 +519,7 @@ kg->kg_estcpu = decay_cpu(loadfac, kg->kg_estcpu); resetpriority(kg); FOREACH_THREAD_IN_GROUP(kg, td) { - if (td->td_priority >= PUSER) { - sched_prio(td, kg->kg_user_pri); - } + resetpriority_thread(td, kg); } } /* end of ksegrp loop */ mtx_unlock_spin(&sched_lock); @@ -561,7 +562,6 @@ newcpu = decay_cpu(loadfac, newcpu); kg->kg_estcpu = newcpu; } - resetpriority(kg); } /* @@ -573,7 +573,6 @@ resetpriority(struct ksegrp *kg) { register unsigned int newpriority; - struct thread *td; if (kg->kg_pri_class == PRI_TIMESHARE) { newpriority = PUSER + kg->kg_estcpu / INVERSE_ESTCPU_WEIGHT + @@ -582,9 +581,25 @@ PRI_MAX_TIMESHARE); kg->kg_user_pri = newpriority; } - FOREACH_THREAD_IN_GROUP(kg, td) { - maybe_resched(td); /* XXXKSE silly */ - } +} + +/* + * Update the thread's priority when the associated ksegroup's user + * priority changes. + */ +static void +resetpriority_thread(struct thread *td, struct ksegrp *kg) +{ + + /* Only change threads with a time sharing user priority. */ + if (td->td_priority < PRI_MIN_TIMESHARE || + td->td_priority > PRI_MAX_TIMESHARE) + return; + + /* XXX the whole needresched thing is broken, but not silly. */ + maybe_resched(td); + + sched_prio(td, kg->kg_user_pri); } /* ARGSUSED */ @@ -674,8 +689,7 @@ kg->kg_estcpu = ESTCPULIM(kg->kg_estcpu + 1); if ((kg->kg_estcpu % INVERSE_ESTCPU_WEIGHT) == 0) { resetpriority(kg); - if (td->td_priority >= PUSER) - td->td_priority = kg->kg_user_pri; + resetpriority_thread(td, kg); } } @@ -735,12 +749,16 @@ sched_nice(struct proc *p, int nice) { struct ksegrp *kg; + struct thread *td; PROC_LOCK_ASSERT(p, MA_OWNED); mtx_assert(&sched_lock, MA_OWNED); p->p_nice = nice; FOREACH_KSEGRP_IN_PROC(p, kg) { resetpriority(kg); + FOREACH_THREAD_IN_GROUP(kg, td) { + resetpriority_thread(td, kg); + } } } @@ -757,14 +775,16 @@ * changing the assignment of a kse to the thread, * and moving a KSE in the system run queue. */ -void -sched_prio(struct thread *td, u_char prio) +static void +sched_priority(struct thread *td, u_char prio) { CTR6(KTR_SCHED, "sched_prio: %p(%s) prio %d newprio %d by %p(%s)", td, td->td_proc->p_comm, td->td_priority, prio, curthread, curthread->td_proc->p_comm); mtx_assert(&sched_lock, MA_OWNED); + if (td->td_priority == prio) + return; if (TD_ON_RUNQ(td)) { adjustrunqueue(td, prio); } else { @@ -772,13 +792,76 @@ } } +/* + * Update a thread's priority when it is lent another thread's + * priority. + */ +void +sched_lend_prio(struct thread *td, u_char prio) +{ + + td->td_flags |= TDF_BORROWING; + sched_priority(td, prio); +} + +/* + * Restore a thread's priority when priority propagation is + * over. The prio argument is the minimum priority the thread + * needs to have to satisfy other possible priority lending + * requests. If the thread's regulary priority is less + * important than prio the thread will keep a priority boost + * of prio. + */ +void +sched_unlend_prio(struct thread *td, u_char prio) +{ + u_char base_pri; + + if (td->td_base_pri >= PRI_MIN_TIMESHARE && + td->td_base_pri <= PRI_MAX_TIMESHARE) + base_pri = td->td_ksegrp->kg_user_pri; + else + base_pri = td->td_base_pri; + if (prio >= base_pri) { + td->td_flags &= ~TDF_BORROWING; + sched_prio(td, base_pri); + } else + sched_lend_prio(td, prio); +} + void +sched_prio(struct thread *td, u_char prio) +{ + u_char oldprio; + + /* First, update the base priority. */ + td->td_base_pri = prio; + + /* + * If the thread is borrowing another thread's priority, don't ever + * lower the priority. + */ + if (td->td_flags & TDF_BORROWING && td->td_priority < prio) + return; + + /* Change the real priority. */ + oldprio = td->td_priority; + sched_priority(td, prio); + + /* + * If the thread is on a turnstile, then let the turnstile update + * its state. + */ + if (TD_ON_LOCK(td) && oldprio != prio) + turnstile_adjust(td, oldprio); +} + +void sched_sleep(struct thread *td) { mtx_assert(&sched_lock, MA_OWNED); td->td_ksegrp->kg_slptime = 0; - td->td_base_pri = td->td_priority; } static void remrunqueue(struct thread *td); @@ -889,8 +972,10 @@ mtx_assert(&sched_lock, MA_OWNED); kg = td->td_ksegrp; - if (kg->kg_slptime > 1) + if (kg->kg_slptime > 1) { updatepri(kg); + resetpriority(kg); + } kg->kg_slptime = 0; setrunqueue(td, SRQ_BORING); } @@ -1157,10 +1242,13 @@ * it here and returning to user mode, so don't waste time setting * it perfectly here. */ + KASSERT((td->td_flags & TDF_BORROWING) == 0, + ("thread with borrowed priority returning to userland")); kg = td->td_ksegrp; if (td->td_priority != kg->kg_user_pri) { mtx_lock_spin(&sched_lock); td->td_priority = kg->kg_user_pri; + td->td_base_pri = kg->kg_user_pri; mtx_unlock_spin(&sched_lock); } } ==== //depot/projects/smpng/sys/kern/sched_ule.c#49 (text+ko) ==== @@ -25,7 +25,7 @@ */ #include -__FBSDID("$FreeBSD: src/sys/kern/sched_ule.c,v 1.143 2004/12/26 23:21:48 jeff Exp $"); +__FBSDID("$FreeBSD: src/sys/kern/sched_ule.c,v 1.144 2004/12/30 20:52:44 jhb Exp $"); #include @@ -46,6 +46,7 @@ #include #include #include +#include #include #ifdef KTRACE #include @@ -133,8 +134,7 @@ #define KEF_XFERABLE 0x0004 /* Thread was added as transferable. */ #define KEF_HOLD 0x0008 /* Thread is temporarily bound. */ #define KEF_REMOVED 0x0010 /* Thread was removed while ASSIGNED */ -#define KEF_PRIOELEV 0x0020 /* Thread has had its prio elevated. */ -#define KEF_INTERNAL 0x0040 +#define KEF_INTERNAL 0x0020 struct kg_sched { struct thread *skg_last_assigned; /* (j) Last thread assigned to */ @@ -234,7 +234,7 @@ #define SCHED_INTERACTIVE(kg) \ (sched_interact_score(kg) < SCHED_INTERACT_THRESH) #define SCHED_CURR(kg, ke) \ - ((ke->ke_flags & KEF_PRIOELEV) || SCHED_INTERACTIVE(kg)) + ((ke->ke_thread->td_flags & TDF_BORROWING) || SCHED_INTERACTIVE(kg)) /* * Cpu percentage computation macros and defines. @@ -315,6 +315,7 @@ static struct kse *sched_choose(void); /* XXX Should be thread * */ static void sched_slice(struct kse *ke); static void sched_priority(struct ksegrp *kg); +static void sched_thread_priority(struct thread *td, u_char prio); static int sched_interact_score(struct ksegrp *kg); static void sched_interact_update(struct ksegrp *kg); static void sched_interact_fork(struct ksegrp *kg); @@ -1066,7 +1067,7 @@ kg = ke->ke_ksegrp; kseq = KSEQ_CPU(ke->ke_cpu); - if (ke->ke_flags & KEF_PRIOELEV) { + if (ke->ke_thread->td_flags & TDF_BORROWING) { ke->ke_slice = SCHED_SLICE_MIN; return; } @@ -1230,7 +1231,7 @@ } void -sched_prio(struct thread *td, u_char prio) +sched_thread_priority(struct thread *td, u_char prio) { struct kse *ke; @@ -1239,6 +1240,8 @@ curthread->td_proc->p_comm); ke = td->td_kse; mtx_assert(&sched_lock, MA_OWNED); + if (td->td_priority == prio) + return; if (TD_ON_RUNQ(td)) { /* * If the priority has been elevated due to priority @@ -1253,8 +1256,6 @@ ke->ke_runq = KSEQ_CPU(ke->ke_cpu)->ksq_curr; runq_add(ke->ke_runq, ke, 0); } - if (prio < td->td_priority) - ke->ke_flags |= KEF_PRIOELEV; /* * Hold this kse on this cpu so that sched_prio() doesn't * cause excessive migration. We only want migration to @@ -1267,7 +1268,71 @@ td->td_priority = prio; } +/* + * Update a thread's priority when it is lent another thread's + * priority. + */ +void +sched_lend_prio(struct thread *td, u_char prio) +{ + + td->td_flags |= TDF_BORROWING; + sched_thread_priority(td, prio); +} + +/* + * Restore a thread's priority when priority propagation is + * over. The prio argument is the minimum priority the thread + * needs to have to satisfy other possible priority lending + * requests. If the thread's regular priority is less + * important than prio, the thread will keep a priority boost + * of prio. + */ void +sched_unlend_prio(struct thread *td, u_char prio) +{ + u_char base_pri; + + if (td->td_base_pri >= PRI_MIN_TIMESHARE && + td->td_base_pri <= PRI_MAX_TIMESHARE) + base_pri = td->td_ksegrp->kg_user_pri; + else + base_pri = td->td_base_pri; + if (prio >= base_pri) { + td->td_flags &= ~ TDF_BORROWING; + sched_thread_priority(td, base_pri); + } else + sched_lend_prio(td, prio); +} + +void +sched_prio(struct thread *td, u_char prio) +{ + u_char oldprio; + + /* First, update the base priority. */ + td->td_base_pri = prio; + + /* + * If the therad is borrowing another thread's priority, don't + * ever lower the priority. + */ + if (td->td_flags & TDF_BORROWING && td->td_priority < prio) + return; + + /* Change the real priority. */ + oldprio = td->td_priority; + sched_thread_priority(td, prio); + + /* + * If the thread is on a turnstile, then let the turnstile update + * its state. + */ + if (TD_ON_LOCK(td) && oldprio != prio) + turnstile_adjust(td, oldprio); +} + +void sched_switch(struct thread *td, struct thread *newtd, int flags) { struct kseq *ksq; @@ -1374,7 +1439,6 @@ mtx_assert(&sched_lock, MA_OWNED); td->td_slptime = ticks; - td->td_base_pri = td->td_priority; } void @@ -1644,21 +1708,14 @@ sched_userret(struct thread *td) { struct ksegrp *kg; - struct kse *ke; - kg = td->td_ksegrp; - ke = td->td_kse; - - if (td->td_priority != kg->kg_user_pri || - ke->ke_flags & KEF_PRIOELEV) { + KASSERT((td->td_flags & TDF_BORROWING) == 0, + ("thread with borrowed priority returning to userland")); + kg = td->td_ksegrp; + if (td->td_priority != kg->kg_user_pri) { mtx_lock_spin(&sched_lock); td->td_priority = kg->kg_user_pri; - if (ke->ke_flags & KEF_PRIOELEV) { - ke->ke_flags &= ~KEF_PRIOELEV; - sched_slice(ke); - if (ke->ke_slice == 0) - mi_switch(SW_INVOL, NULL); - } + td->td_base_pri = kg->kg_user_pri; mtx_unlock_spin(&sched_lock); } } ==== //depot/projects/smpng/sys/kern/subr_trap.c#75 (text+ko) ==== @@ -38,7 +38,7 @@ */ #include -__FBSDID("$FreeBSD: src/sys/kern/subr_trap.c,v 1.277 2004/12/26 07:30:35 jeff Exp $"); +__FBSDID("$FreeBSD: src/sys/kern/subr_trap.c,v 1.278 2004/12/30 20:30:58 jhb Exp $"); #include "opt_ktrace.h" #include "opt_mac.h" @@ -128,6 +128,7 @@ ticks = td->td_sticks - oticks; addupc_task(td, TRAPF_PC(frame), (u_int)ticks * psratio); } + /* * Let the scheduler adjust our priority etc. */ ==== //depot/projects/smpng/sys/kern/subr_turnstile.c#18 (text+ko) ==== @@ -59,7 +59,7 @@ #include "opt_turnstile_profiling.h" #include -__FBSDID("$FreeBSD: src/sys/kern/subr_turnstile.c,v 1.150 2004/10/12 18:36:20 jhb Exp $"); +__FBSDID("$FreeBSD: src/sys/kern/subr_turnstile.c,v 1.151 2004/12/30 20:52:44 jhb Exp $"); #include #include @@ -145,7 +145,9 @@ #ifdef TURNSTILE_PROFILING static void init_turnstile_profiling(void *arg); #endif -static void propagate_priority(struct thread *); +static void propagate_priority(struct thread *td); +static int turnstile_adjust_thread(struct turnstile *ts, + struct thread *td); static void turnstile_setowner(struct turnstile *ts, struct thread *owner); /* @@ -158,7 +160,6 @@ { struct turnstile_chain *tc; struct turnstile *ts; - struct thread *td1; int pri; mtx_assert(&sched_lock, MA_OWNED); @@ -187,8 +188,8 @@ * isn't SRUN or SLOCK. */ KASSERT(!TD_IS_SLEEPING(td), - ("sleeping thread (pid %d) owns a non-sleepable lock", - td->td_proc->p_pid)); + ("sleeping thread (tid %d) owns a non-sleepable lock", + td->td_tid)); /* * If this thread already has higher priority than the @@ -198,10 +199,16 @@ return; /* - * If lock holder is actually running, just bump priority. + * Bump this thread's priority. + */ + sched_lend_prio(td, pri); + + /* + * If lock holder is actually running or on the run queue + * then we are done. */ - if (TD_IS_RUNNING(td)) { - td->td_priority = pri; + if (TD_IS_RUNNING(td) || TD_ON_RUNQ(td)) { + MPASS(td->td_blocked == NULL); return; } @@ -214,27 +221,11 @@ #endif /* - * If on run queue move to new run queue, and quit. - * XXXKSE this gets a lot more complicated under threads - * but try anyhow. - */ - if (TD_ON_RUNQ(td)) { - MPASS(td->td_blocked == NULL); - sched_prio(td, pri); - return; - } - - /* - * Bump this thread's priority. - */ - td->td_priority = pri; - - /* * If we aren't blocked on a lock, we should be. */ KASSERT(TD_ON_LOCK(td), ( - "process %d(%s):%d holds %s but isn't blocked on a lock\n", - td->td_proc->p_pid, td->td_proc->p_comm, td->td_state, + "thread %d(%s):%d holds %s but isn't blocked on a lock\n", + td->td_tid, td->td_proc->p_comm, td->td_state, ts->ts_lockobj->lo_name)); /* @@ -245,61 +236,81 @@ tc = TC_LOOKUP(ts->ts_lockobj); mtx_lock_spin(&tc->tc_lock); - /* - * This thread may not be blocked on this turnstile anymore - * but instead might already be woken up on another CPU - * that is waiting on sched_lock in turnstile_unpend() to - * finish waking this thread up. We can detect this case - * by checking to see if this thread has been given a - * turnstile by either turnstile_signal() or - * turnstile_broadcast(). In this case, treat the thread as - * if it was already running. - */ - if (td->td_turnstile != NULL) { + /* Resort td on the list if needed. */ + if (!turnstile_adjust_thread(ts, td)) { mtx_unlock_spin(&tc->tc_lock); return; } + mtx_unlock_spin(&tc->tc_lock); + } +} + +/* + * Adjust the thread's position on a turnstile after its priority has been + * changed. + */ +static int +turnstile_adjust_thread(struct turnstile *ts, struct thread *td) +{ + struct turnstile_chain *tc; + struct thread *td1, *td2; + + mtx_assert(&sched_lock, MA_OWNED); + MPASS(TD_ON_LOCK(td)); - /* - * Check if the thread needs to be moved up on - * the blocked chain. It doesn't need to be moved - * if it is already at the head of the list or if - * the item in front of it still has a higher priority. - */ - if (td == TAILQ_FIRST(&ts->ts_blocked)) { - mtx_unlock_spin(&tc->tc_lock); - continue; - } + /* + * This thread may not be blocked on this turnstile anymore + * but instead might already be woken up on another CPU + * that is waiting on sched_lock in turnstile_unpend() to + * finish waking this thread up. We can detect this case + * by checking to see if this thread has been given a + * turnstile by either turnstile_signal() or + * turnstile_broadcast(). In this case, treat the thread as + * if it was already running. + */ + if (td->td_turnstile != NULL) + return (0); - td1 = TAILQ_PREV(td, threadqueue, td_lockq); - if (td1->td_priority <= pri) { - mtx_unlock_spin(&tc->tc_lock); - continue; - } + /* + * Check if the thread needs to be moved on the blocked chain. + * It needs to be moved if either its priority is lower than + * the previous thread or higher than the next thread. + */ + tc = TC_LOOKUP(ts->ts_lockobj); + mtx_assert(&tc->tc_lock, MA_OWNED); + td1 = TAILQ_PREV(td, threadqueue, td_lockq); + td2 = TAILQ_NEXT(td, td_lockq); + if ((td1 != NULL && td->td_priority < td1->td_priority) || + (td2 != NULL && td->td_priority > td2->td_priority)) { /* * Remove thread from blocked chain and determine where - * it should be moved up to. Since we know that td1 has - * a lower priority than td, we know that at least one - * thread in the chain has a lower priority and that - * td1 will thus not be NULL after the loop. + * it should be moved to. */ mtx_lock_spin(&td_contested_lock); TAILQ_REMOVE(&ts->ts_blocked, td, td_lockq); TAILQ_FOREACH(td1, &ts->ts_blocked, td_lockq) { MPASS(td1->td_proc->p_magic == P_MAGIC); - if (td1->td_priority > pri) + if (td1->td_priority > td->td_priority) break; } - MPASS(td1 != NULL); - TAILQ_INSERT_BEFORE(td1, td, td_lockq); + if (td1 == NULL) + TAILQ_INSERT_TAIL(&ts->ts_blocked, td, td_lockq); + else + TAILQ_INSERT_BEFORE(td1, td, td_lockq); mtx_unlock_spin(&td_contested_lock); - CTR4(KTR_LOCK, - "propagate_priority: td %p moved before %p on [%p] %s", - td, td1, ts->ts_lockobj, ts->ts_lockobj->lo_name); - mtx_unlock_spin(&tc->tc_lock); + if (td1 == NULL) + CTR3(KTR_LOCK, + "turnstile_adjust_thread: td %d put at tail on [%p] %s", + td->td_tid, ts->ts_lockobj, ts->ts_lockobj->lo_name); + else + CTR4(KTR_LOCK, + "turnstile_adjust_thread: td %d moved before %d on [%p] %s", + td->td_tid, td1->td_tid, ts->ts_lockobj, + ts->ts_lockobj->lo_name); } + return (1); } /* @@ -355,6 +366,46 @@ SYSINIT(turnstile0, SI_SUB_LOCK, SI_ORDER_ANY, init_turnstile0, NULL); /* + * Update a thread on the turnstile list after it's priority has been changed. + * The old priority is passed in as an argument. + */ +void +turnstile_adjust(struct thread *td, u_char oldpri) +{ + struct turnstile_chain *tc; + struct turnstile *ts; + + mtx_assert(&sched_lock, MA_OWNED); + MPASS(TD_ON_LOCK(td)); + + /* + * Pick up the lock that td is blocked on. + */ + ts = td->td_blocked; + MPASS(ts != NULL); + tc = TC_LOOKUP(ts->ts_lockobj); + mtx_lock_spin(&tc->tc_lock); + + /* Resort the turnstile on the list. */ + if (!turnstile_adjust_thread(ts, td)) { + mtx_unlock_spin(&tc->tc_lock); + return; + } + + /* + * If our priority was lowered and we are at the head of the + * turnstile, then propagate our new priority up the chain. + * Note that we currently don't try to revoke lent priorities + * when our priority goes up. + */ + if (td == TAILQ_FIRST(&ts->ts_blocked) && td->td_priority < oldpri) { + mtx_unlock_spin(&tc->tc_lock); + propagate_priority(td); + } else + mtx_unlock_spin(&tc->tc_lock); +} + +/* * Set the owner of the lock this turnstile is attached to. */ static void @@ -470,7 +521,7 @@ */ mtx_lock_spin(&sched_lock); if (td->td_priority < owner->td_priority) - owner->td_priority = td->td_priority; + sched_lend_prio(owner, td->td_priority); mtx_unlock_spin(&sched_lock); } @@ -578,14 +629,14 @@ propagate_priority(td); if (LOCK_LOG_TEST(lock, 0)) - CTR4(KTR_LOCK, "%s: td %p blocked on [%p] %s", __func__, td, - lock, lock->lo_name); >>> TRUNCATED FOR MAIL (1000 lines) <<<