Date: Sat, 1 Dec 2007 22:00:37 GMT From: Peter Wemm <peter@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 129905 for review Message-ID: <200712012200.lB1M0bWG078395@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=129905 Change 129905 by peter@peter_daintree on 2007/12/01 21:59:45 IFC Affected files ... .. //depot/projects/bike_sched/sys/kern/kern_thread.c#5 integrate Differences ... ==== //depot/projects/bike_sched/sys/kern/kern_thread.c#5 (text+ko) ==== @@ -27,7 +27,7 @@ */ #include <sys/cdefs.h> -__FBSDID("$FreeBSD: src/sys/kern/kern_thread.c,v 1.234 2006/06/30 08:10:55 maxim Exp $"); +__FBSDID("$FreeBSD: src/sys/kern/kern_thread.c,v 1.262 2007/11/15 14:20:06 rrs Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -49,33 +49,26 @@ #include <vm/vm.h> #include <vm/vm_extern.h> #include <vm/uma.h> +#include <sys/eventhandler.h> /* * thread related storage. */ static uma_zone_t thread_zone; -/* DEBUG ONLY */ SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); -static int thread_debug = 0; -SYSCTL_INT(_kern_threads, OID_AUTO, debug, CTLFLAG_RW, - &thread_debug, 0, "thread debug"); int max_threads_per_proc = 1500; SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW, &max_threads_per_proc, 0, "Limit on threads per proc"); -int max_groups_per_proc = 1500; -SYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW, - &max_groups_per_proc, 0, "Limit on thread groups per proc"); - int max_threads_hits; SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD, &max_threads_hits, 0, ""); TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); -struct mtx kse_zombie_lock; -MTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN); +static struct mtx zombie_lock; +MTX_SYSINIT(zombie_lock, &zombie_lock, "zombie lock", MTX_SPIN); struct mtx tid_lock; static struct unrhdr *tid_unrhdr; @@ -93,24 +86,19 @@ td->td_oncpu = NOCPU; td->td_tid = alloc_unr(tid_unrhdr); + td->td_syscalls = 0; /* * Note that td_critnest begins life as 1 because the thread is not * running and is thereby implicitly waiting to be on the receiving - * end of a context switch. A context switch must occur inside a - * critical section, and in fact, includes hand-off of the sched_lock. - * After a context switch to a newly created thread, it will release - * sched_lock for the first time, and its td_critnest will hit 0 for - * the first time. This happens on the far end of a context switch, - * and when it context switches away from itself, it will in fact go - * back into a critical section, and hand off the sched lock to the - * next thread. + * end of a context switch. */ td->td_critnest = 1; - + EVENTHANDLER_INVOKE(thread_ctor, td); #ifdef AUDIT audit_thread_alloc(td); #endif + umtx_thread_alloc(td); return (0); } @@ -147,6 +135,7 @@ #ifdef AUDIT audit_thread_free(td); #endif + EVENTHANDLER_INVOKE(thread_dtor, td); free_unr(tid_unrhdr, td->td_tid); sched_newthread(td); } @@ -161,13 +150,13 @@ td = (struct thread *)mem; - vm_thread_new(td, 0); - cpu_thread_setup(td); td->td_sleepqueue = sleepq_alloc(); td->td_turnstile = turnstile_alloc(); - td->td_umtxq = umtxq_alloc(); + EVENTHANDLER_INVOKE(thread_init, td); td->td_sched = (struct td_sched *)&td[1]; sched_newthread(td); + umtx_thread_init(td); + td->td_kstack = 0; return (0); } @@ -180,10 +169,10 @@ struct thread *td; td = (struct thread *)mem; + EVENTHANDLER_INVOKE(thread_fini, td); turnstile_free(td->td_turnstile); sleepq_free(td->td_sleepqueue); - umtxq_free(td->td_umtxq); - vm_thread_dispose(td); + umtx_thread_fini(td); } /* @@ -195,11 +184,16 @@ * proc_init() */ void +proc_linkup0(struct proc *p, struct thread *td) +{ + TAILQ_INIT(&p->p_threads); /* all threads in proc */ + proc_linkup(p, td); +} + +void proc_linkup(struct proc *p, struct thread *td) { - TAILQ_INIT(&p->p_threads); /* all threads in proc */ - TAILQ_INIT(&p->p_suspended); /* Threads suspended */ sigqueue_init(&p->p_sigqueue, p); p->p_ksi = ksiginfo_alloc(1); if (p->p_ksi != NULL) { @@ -208,7 +202,6 @@ } LIST_INIT(&p->p_mqnotifier); p->p_numthreads = 0; - thread_link(td, p); } @@ -224,19 +217,20 @@ thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), thread_ctor, thread_dtor, thread_init, thread_fini, - UMA_ALIGN_CACHE, 0); + 16 - 1, 0); } /* - * Stash an embarasingly extra thread into the zombie thread queue. + * Place an unused thread on the zombie list. + * Use the slpq as that must be unused by now. */ void -thread_stash(struct thread *td) +thread_zombie(struct thread *td) { - mtx_lock_spin(&kse_zombie_lock); - TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq); - mtx_unlock_spin(&kse_zombie_lock); + mtx_lock_spin(&zombie_lock); + TAILQ_INSERT_HEAD(&zombie_threads, td, td_slpq); + mtx_unlock_spin(&zombie_lock); } /* @@ -252,19 +246,22 @@ * we really don't care about the next instant.. */ if (!TAILQ_EMPTY(&zombie_threads)) { - mtx_lock_spin(&kse_zombie_lock); + mtx_lock_spin(&zombie_lock); td_first = TAILQ_FIRST(&zombie_threads); if (td_first) TAILQ_INIT(&zombie_threads); - mtx_unlock_spin(&kse_zombie_lock); + mtx_unlock_spin(&zombie_lock); while (td_first) { - td_next = TAILQ_NEXT(td_first, td_runq); + td_next = TAILQ_NEXT(td_first, td_slpq); if (td_first->td_ucred) crfree(td_first->td_ucred); thread_free(td_first); td_first = td_next; } } +#ifdef KSE + upcall_reap(); +#endif } /* @@ -273,19 +270,31 @@ struct thread * thread_alloc(void) { + struct thread *td; thread_reap(); /* check if any zombies to get */ - return (uma_zalloc(thread_zone, M_WAITOK)); + + td = (struct thread *)uma_zalloc(thread_zone, M_WAITOK); + KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack")); + if (!vm_thread_new(td, 0)) { + uma_zfree(thread_zone, td); + return (NULL); + } + cpu_thread_alloc(td); + return (td); } -/* * Deallocate a thread. */ void thread_free(struct thread *td) { - cpu_thread_clean(td); + cpu_thread_free(td); + if (td->td_altkstack != 0) + vm_thread_dispose_altkstack(td); + if (td->td_kstack != 0) + vm_thread_dispose(td); uma_zfree(thread_zone, td); } @@ -321,23 +330,27 @@ { uint64_t new_switchtime; struct thread *td; + struct thread *td2; struct proc *p; td = curthread; p = td->td_proc; - mtx_assert(&sched_lock, MA_OWNED); + PROC_SLOCK_ASSERT(p, MA_OWNED); mtx_assert(&Giant, MA_NOTOWNED); + PROC_LOCK_ASSERT(p, MA_OWNED); KASSERT(p != NULL, ("thread exiting without a process")); CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td, - (long)p->p_pid, p->p_comm); + (long)p->p_pid, td->td_name); KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending")); #ifdef AUDIT AUDIT_SYSCALL_EXIT(0, td); #endif + umtx_thread_exit(td); + /* * drop FPU & debug register state storage, or any other * architecture specific resources that @@ -348,17 +361,12 @@ /* Do the same timestamp bookkeeping that mi_switch() would do. */ new_switchtime = cpu_ticks(); p->p_rux.rux_runtime += (new_switchtime - PCPU_GET(switchtime)); - p->p_rux.rux_uticks += td->td_uticks; - p->p_rux.rux_sticks += td->td_sticks; - p->p_rux.rux_iticks += td->td_iticks; PCPU_SET(switchtime, new_switchtime); PCPU_SET(switchticks, ticks); - cnt.v_swtch++; - - /* Add our usage into the usage of all our children. */ - if (p->p_numthreads == 1) - ruadd(p->p_ru, &p->p_rux, &p->p_stats->p_cru, &p->p_crux); - + PCPU_INC(cnt.v_swtch); + /* Save our resource usage in our process. */ + td->td_ru.ru_nvcsw++; + rucollect(&p->p_ru, &td->td_ru); /* * The last thread is left attached to the process * So that the whole bundle gets recycled. Skip @@ -369,7 +377,9 @@ */ if (p->p_flag & P_HADTHREADS) { if (p->p_numthreads > 1) { + thread_lock(td); thread_unlink(td); + thread_unlock(td); sched_exit(p, td); /* @@ -378,10 +388,13 @@ * in exit1() after it is the only survivor. */ if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { - if (p->p_numthreads == p->p_suspcount) + if (p->p_numthreads == p->p_suspcount) { + thread_lock(p->p_singlethread); thread_unsuspend_one(p->p_singlethread); + thread_unlock(p->p_singlethread); + } } - PROC_UNLOCK(p); + atomic_add_int(&td->td_proc->p_exitthreads, 1); PCPU_SET(deadthread, td); } else { /* @@ -395,17 +408,15 @@ */ panic ("thread_exit: Last thread exiting on its own"); } - } else { - /* - * non threaded process comes here. - * This includes an EX threaded process that is coming - * here via exit1(). (exit1 dethreads the proc first). - */ - PROC_UNLOCK(p); - } + } + PROC_UNLOCK(p); + thread_lock(td); + /* Save our tick information with both the thread and proc locked */ + ruxagg(&p->p_rux, td); + PROC_SUNLOCK(p); td->td_state = TDS_INACTIVE; CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td); - cpu_throw(td, choosethread()); + sched_throw(td); panic("I'm a teapot!"); /* NOTREACHED */ } @@ -421,10 +432,15 @@ mtx_assert(&Giant, MA_NOTOWNED); KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()")); - FOREACH_THREAD_IN_PROC(p, td) { - cpu_thread_clean(td); - crfree(td->td_ucred); - } + td = FIRST_THREAD_IN_PROC(p); + /* Lock the last thread so we spin until it exits cpu_throw(). */ + thread_lock(td); + thread_unlock(td); + /* Wait for any remaining threads to exit cpu_throw(). */ + while (p->p_exitthreads) + sched_relinquish(curthread); + cpu_thread_clean(td); + crfree(td->td_ucred); thread_reap(); /* check for zombie threads etc. */ } @@ -443,9 +459,14 @@ thread_link(struct thread *td, struct proc *p) { + /* + * XXX This can't be enabled because it's called for proc0 before + * it's spinlock has been created. + * PROC_SLOCK_ASSERT(p, MA_OWNED); + */ td->td_state = TDS_INACTIVE; td->td_proc = p; - td->td_flags = 0; + td->td_flags = TDF_INMEM; LIST_INIT(&td->td_contested); sigqueue_init(&td->td_sigqueue, p); @@ -466,7 +487,13 @@ struct proc *p = td->td_proc; KASSERT((p->p_numthreads == 1), ("Unthreading with >1 threads")); +#ifdef KSE + thread_lock(td); + thread_unlock(td); + thread_zombie(td->td_standin); +#else p->p_flag &= ~P_HADTHREADS; +#endif } /* @@ -478,7 +505,7 @@ { struct proc *p = td->td_proc; - mtx_assert(&sched_lock, MA_OWNED); + PROC_SLOCK_ASSERT(p, MA_OWNED); TAILQ_REMOVE(&p->p_threads, td, td_plist); p->p_numthreads--; /* could clear a few other things here */ @@ -530,7 +557,7 @@ p->p_flag &= ~P_SINGLE_BOUNDARY; } p->p_flag |= P_STOPPED_SINGLE; - mtx_lock_spin(&sched_lock); + PROC_SLOCK(p); p->p_singlethread = td; if (mode == SINGLE_EXIT) remaining = p->p_numthreads; @@ -544,6 +571,7 @@ FOREACH_THREAD_IN_PROC(p, td2) { if (td2 == td) continue; + thread_lock(td2); td2->td_flags |= TDF_ASTPENDING; if (TD_IS_INHIBITED(td2)) { switch (mode) { @@ -557,18 +585,14 @@ sleepq_abort(td2, EINTR); break; case SINGLE_BOUNDARY: - if (TD_IS_SUSPENDED(td2) && - !(td2->td_flags & TDF_BOUNDARY)) - thread_unsuspend_one(td2); - if (TD_ON_SLEEPQ(td2) && - (td2->td_flags & TDF_SINTR)) - sleepq_abort(td2, ERESTART); break; default: - if (TD_IS_SUSPENDED(td2)) + if (TD_IS_SUSPENDED(td2)) { + thread_unlock(td2); continue; + } /* - * maybe other inhibitted states too? + * maybe other inhibited states too? */ if ((td2->td_flags & TDF_SINTR) && (td2->td_inhibitors & @@ -582,6 +606,7 @@ forward_signal(td2); } #endif + thread_unlock(td2); } if (mode == SINGLE_EXIT) remaining = p->p_numthreads; @@ -601,13 +626,7 @@ * Wake us up when everyone else has suspended. * In the mean time we suspend as well. */ - thread_stopped(p); - thread_suspend_one(td); - PROC_UNLOCK(p); - mi_switch(SW_VOL, NULL); - mtx_unlock_spin(&sched_lock); - PROC_LOCK(p); - mtx_lock_spin(&sched_lock); + thread_suspend_switch(td); if (mode == SINGLE_EXIT) remaining = p->p_numthreads; else if (mode == SINGLE_BOUNDARY) @@ -626,7 +645,7 @@ p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT); thread_unthread(td); } - mtx_unlock_spin(&sched_lock); + PROC_SUNLOCK(p); return (0); } @@ -699,7 +718,7 @@ if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) sigqueue_flush(&td->td_sigqueue); - mtx_lock_spin(&sched_lock); + PROC_SLOCK(p); thread_stopped(p); /* * If the process is waiting for us to exit, @@ -708,44 +727,75 @@ */ if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) thread_exit(); - + if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { + if (p->p_numthreads == p->p_suspcount + 1) { + thread_lock(p->p_singlethread); + thread_unsuspend_one(p->p_singlethread); + thread_unlock(p->p_singlethread); + } + } + PROC_UNLOCK(p); + thread_lock(td); /* * When a thread suspends, it just - * moves to the processes's suspend queue - * and stays there. + * gets taken off all queues. */ thread_suspend_one(td); if (return_instead == 0) { p->p_boundary_count++; td->td_flags |= TDF_BOUNDARY; } - if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { - if (p->p_numthreads == p->p_suspcount) - thread_unsuspend_one(p->p_singlethread); - } - PROC_UNLOCK(p); + PROC_SUNLOCK(p); mi_switch(SW_INVOL, NULL); - if (return_instead == 0) { - p->p_boundary_count--; + if (return_instead == 0) td->td_flags &= ~TDF_BOUNDARY; - } - mtx_unlock_spin(&sched_lock); + thread_unlock(td); PROC_LOCK(p); + if (return_instead == 0) + p->p_boundary_count--; } return (0); } void +thread_suspend_switch(struct thread *td) +{ + struct proc *p; + + p = td->td_proc; + KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); + PROC_LOCK_ASSERT(p, MA_OWNED); + PROC_SLOCK_ASSERT(p, MA_OWNED); + /* + * We implement thread_suspend_one in stages here to avoid + * dropping the proc lock while the thread lock is owned. + */ + thread_stopped(p); + p->p_suspcount++; + PROC_UNLOCK(p); + thread_lock(td); + sched_sleep(td); + TD_SET_SUSPENDED(td); + PROC_SUNLOCK(p); + DROP_GIANT(); + mi_switch(SW_VOL, NULL); + thread_unlock(td); + PICKUP_GIANT(); + PROC_LOCK(p); + PROC_SLOCK(p); +} + +void thread_suspend_one(struct thread *td) { struct proc *p = td->td_proc; - mtx_assert(&sched_lock, MA_OWNED); - PROC_LOCK_ASSERT(p, MA_OWNED); + PROC_SLOCK_ASSERT(p, MA_OWNED); + THREAD_LOCK_ASSERT(td, MA_OWNED); KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); p->p_suspcount++; + sched_sleep(td); TD_SET_SUSPENDED(td); - TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq); } void @@ -753,9 +803,9 @@ { struct proc *p = td->td_proc; - mtx_assert(&sched_lock, MA_OWNED); - PROC_LOCK_ASSERT(p, MA_OWNED); - TAILQ_REMOVE(&p->p_suspended, td, td_runq); + PROC_SLOCK_ASSERT(p, MA_OWNED); + THREAD_LOCK_ASSERT(td, MA_OWNED); + KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended")); TD_CLR_SUSPENDED(td); p->p_suspcount--; setrunnable(td); @@ -769,11 +819,15 @@ { struct thread *td; - mtx_assert(&sched_lock, MA_OWNED); PROC_LOCK_ASSERT(p, MA_OWNED); + PROC_SLOCK_ASSERT(p, MA_OWNED); if (!P_SHOULDSTOP(p)) { - while ((td = TAILQ_FIRST(&p->p_suspended))) { - thread_unsuspend_one(td); + FOREACH_THREAD_IN_PROC(p, td) { + thread_lock(td); + if (TD_IS_SUSPENDED(td)) { + thread_unsuspend_one(td); + } + thread_unlock(td); } } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && (p->p_numthreads == p->p_suspcount)) { @@ -782,7 +836,9 @@ * threading request. Now we've downgraded to single-threaded, * let it continue. */ + thread_lock(p->p_singlethread); thread_unsuspend_one(p->p_singlethread); + thread_unlock(p->p_singlethread); } } @@ -799,7 +855,7 @@ p = td->td_proc; PROC_LOCK_ASSERT(p, MA_OWNED); p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY); - mtx_lock_spin(&sched_lock); + PROC_SLOCK(p); p->p_singlethread = NULL; /* * If there are other threads they mey now run, @@ -808,11 +864,15 @@ * to continue however as this is a bad place to stop. */ if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { - while ((td = TAILQ_FIRST(&p->p_suspended))) { - thread_unsuspend_one(td); + FOREACH_THREAD_IN_PROC(p, td) { + thread_lock(td); + if (TD_IS_SUSPENDED(td)) { + thread_unsuspend_one(td); + } + thread_unlock(td); } } - mtx_unlock_spin(&sched_lock); + PROC_SUNLOCK(p); } struct thread * @@ -821,11 +881,11 @@ struct thread *td; PROC_LOCK_ASSERT(p, MA_OWNED); - mtx_lock_spin(&sched_lock); + PROC_SLOCK(p); FOREACH_THREAD_IN_PROC(p, td) { if (td->td_tid == tid) break; } - mtx_unlock_spin(&sched_lock); + PROC_SUNLOCK(p); return (td); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200712012200.lB1M0bWG078395>