Date: Sun, 7 Nov 2010 13:41:18 +0000 (UTC) From: David Xu <davidxu@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r214914 - in user/davidxu/libthr/sys: kern sys Message-ID: <201011071341.oA7DfILV048093@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: davidxu Date: Sun Nov 7 13:41:17 2010 New Revision: 214914 URL: http://svn.freebsd.org/changeset/base/214914 Log: Implement robust mutex. userland calls kernel to lock and unlock robust mutex. The kernel remember a list of robust mutexes, and when a thread exits, if its robust mutexes are not unlocked, the kernel will atomatically unlock them and set their robust states to OWNERDEAD. Modified: user/davidxu/libthr/sys/kern/kern_thr.c user/davidxu/libthr/sys/kern/kern_thread.c user/davidxu/libthr/sys/kern/kern_umtx.c user/davidxu/libthr/sys/sys/_umtx.h user/davidxu/libthr/sys/sys/umtx.h Modified: user/davidxu/libthr/sys/kern/kern_thr.c ============================================================================== --- user/davidxu/libthr/sys/kern/kern_thr.c Sun Nov 7 12:29:26 2010 (r214913) +++ user/davidxu/libthr/sys/kern/kern_thr.c Sun Nov 7 13:41:17 2010 (r214914) @@ -285,12 +285,10 @@ thr_exit(struct thread *td, struct thr_e kern_umtx_wake(td, uap->state, INT_MAX, 0); } + umtx_thread_exit(td); + rw_wlock(&tidhash_lock); PROC_LOCK(p); - /* - * Shutting down last thread in the proc. This will actually - * call exit() in the trampoline when it returns. - */ if (p->p_numthreads != 1) { LIST_REMOVE(td, td_hash); rw_wunlock(&tidhash_lock); @@ -299,9 +297,11 @@ thr_exit(struct thread *td, struct thr_e thread_stopped(p); thread_exit(); /* NOTREACHED */ + } else { + PROC_UNLOCK(p); + rw_wunlock(&tidhash_lock); + exit1(td, 0); } - PROC_UNLOCK(p); - rw_wunlock(&tidhash_lock); return (0); } Modified: user/davidxu/libthr/sys/kern/kern_thread.c ============================================================================== --- user/davidxu/libthr/sys/kern/kern_thread.c Sun Nov 7 12:29:26 2010 (r214913) +++ user/davidxu/libthr/sys/kern/kern_thread.c Sun Nov 7 13:41:17 2010 (r214914) @@ -373,7 +373,6 @@ thread_exit(void) #ifdef AUDIT AUDIT_SYSCALL_EXIT(0, td); #endif - umtx_thread_exit(td); /* * drop FPU & debug register state storage, or any other * architecture specific resources that @@ -754,6 +753,7 @@ thread_suspend_check(int return_instead) if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { PROC_UNLOCK(p); tidhash_remove(td); + umtx_thread_exit(td); PROC_LOCK(p); tdsigcleanup(td); PROC_SLOCK(p); Modified: user/davidxu/libthr/sys/kern/kern_umtx.c ============================================================================== --- user/davidxu/libthr/sys/kern/kern_umtx.c Sun Nov 7 12:29:26 2010 (r214913) +++ user/davidxu/libthr/sys/kern/kern_umtx.c Sun Nov 7 13:41:17 2010 (r214914) @@ -116,6 +116,16 @@ struct umtx_pi { struct umtx_key pi_key; }; +struct robust_info { + struct thread *ownertd; + SLIST_ENTRY(robust_info) hash_qe; + LIST_ENTRY(robust_info) td_qe; + struct umutex *umtxp; +}; + +SLIST_HEAD(robust_hashlist, robust_info); +LIST_HEAD(robust_list, robust_info); + /* A userland synchronous object user. */ struct umtx_q { /* Linked list for the hash. */ @@ -154,6 +164,8 @@ struct umtx_q { struct umtxq_queue *uq_cur_queue; int uq_repair_mutex; + + struct robust_list uq_rob_list; }; TAILQ_HEAD(umtxq_head, umtx_q); @@ -195,6 +207,13 @@ struct umtxq_chain { }; +struct robust_chain { + /* Lock for this chain. */ + struct mtx lock; + struct robust_hashlist rob_list; +}; + + #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED) #define UMTXQ_BUSY_ASSERT(uc) KASSERT(&(uc)->uc_busy, ("umtx chain is not busy")) @@ -224,7 +243,11 @@ struct umtxq_chain { #define BUSY_SPINS 200 +#define ROBUST_CHAINS 128 +#define ROBUST_SHIFTS (__WORD_BIT - 7) + static uma_zone_t umtx_pi_zone; +static uma_zone_t robust_zone; static struct umtxq_chain umtxq_chains[2][UMTX_CHAINS]; static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory"); static int umtx_pi_allocated; @@ -234,6 +257,8 @@ static int umtx_cvsig_migrate = 0; static int umtx_cvsig_migrate = 1; #endif +static struct robust_chain robust_chains[ROBUST_CHAINS]; + SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug"); SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD, &umtx_pi_allocated, 0, "Allocated umtx_pi"); @@ -284,10 +309,17 @@ static void umtx_key_release(struct umtx static struct umtx_pi *umtx_pi_alloc(int); static void umtx_pi_free(struct umtx_pi *pi); static void umtx_pi_adjust_locked(struct thread *td, u_char oldpri); -static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags); -static void umtx_thread_cleanup(struct thread *td); +static int do_unlock_pp(struct thread *, struct umutex *, uint32_t, int); +static void umtx_thread_cleanup(struct thread *); static void umtx_exec_hook(void *arg __unused, struct proc *p __unused, struct image_params *imgp __unused); +static void umtx_exit_hook(void *arg __unused, struct proc *p __unused); +static struct robust_info *robust_alloc(void); +static void robust_free(struct robust_info *); +static int robust_insert(struct thread *, struct robust_info *); +static void robust_remove(struct thread *, struct umutex *); +static int do_unlock_umutex(struct thread *, struct umutex *, int); + SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL); #define umtxq_signal(key, nwake) umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE) @@ -303,6 +335,8 @@ umtxq_sysinit(void *arg __unused) umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); + robust_zone = uma_zcreate("robust umtx", sizeof(struct robust_info), + NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); for (i = 0; i < 2; ++i) { for (j = 0; j < UMTX_CHAINS; ++j) { mtx_init(&umtxq_chains[i][j].uc_lock, "umtxql", NULL, @@ -315,9 +349,18 @@ umtxq_sysinit(void *arg __unused) umtxq_chains[i][j].uc_waiters = 0; } } + + for (i = 0; i < ROBUST_CHAINS; ++i) { + mtx_init(&robust_chains[i].lock, "robql", NULL, + MTX_DEF | MTX_DUPOK); + SLIST_INIT(&robust_chains[i].rob_list); + } + mtx_init(&umtx_lock, "umtx lock", NULL, MTX_SPIN); EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL, EVENTHANDLER_PRI_ANY); + EVENTHANDLER_REGISTER(process_exit, umtx_exit_hook, NULL, + EVENTHANDLER_PRI_ANY); } struct umtx_q * @@ -326,9 +369,11 @@ umtxq_alloc(void) struct umtx_q *uq; uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO); - uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX, M_WAITOK | M_ZERO); + uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX, + M_WAITOK | M_ZERO); TAILQ_INIT(&uq->uq_spare_queue->head); TAILQ_INIT(&uq->uq_pi_contested); + LIST_INIT(&uq->uq_rob_list); uq->uq_inherited_pri = PRI_MAX; return (uq); } @@ -1321,6 +1366,19 @@ _do_lock_normal(struct thread *td, struc return (0); } +static void +update_robst(struct umutex *m, int td_exit) +{ + uint32_t robst = fubyte(&m->m_robstate); + + if (robst == UMUTEX_ROBST_NORMAL) { + if (td_exit) + subyte(&m->m_robstate, UMUTEX_ROBST_OWNERDEAD); + } else if (!td_exit && robst == UMUTEX_ROBST_INCONSISTENT) { + subyte(&m->m_robstate, UMUTEX_ROBST_NOTRECOVERABLE); + } +} + /* * Lock PTHREAD_PRIO_NONE protocol POSIX mutex. */ @@ -1328,7 +1386,8 @@ _do_lock_normal(struct thread *td, struc * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex. */ static int -do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags) +do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags, + int td_exit) { struct umtx_key key; uint32_t owner, old, id; @@ -1349,6 +1408,9 @@ do_unlock_normal(struct thread *td, stru if ((owner & ~UMUTEX_CONTESTED) != id) return (EPERM); + if ((flags & UMUTEX_ROBUST) != 0) + update_robst(m, td_exit); + if ((owner & UMUTEX_CONTESTED) == 0) { old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED); if (old == -1) @@ -1974,7 +2036,8 @@ _do_lock_pi(struct thread *td, struct um * Unlock a PI mutex. */ static int -do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags) +do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags, + int td_exit) { struct umtx_key key; struct umtx_q *uq_first, *uq_first2, *uq_me; @@ -1995,6 +2058,9 @@ do_unlock_pi(struct thread *td, struct u if ((owner & ~UMUTEX_CONTESTED) != id) return (EPERM); + if ((flags & UMUTEX_ROBUST) != 0) + update_robst(m, td_exit); + /* This should be done in userland */ if ((owner & UMUTEX_CONTESTED) == 0) { old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED); @@ -2207,7 +2273,8 @@ out: * Unlock a PP mutex. */ static int -do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags) +do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags, + int td_exit) { struct umtx_key key; struct umtx_q *uq, *uq2; @@ -2237,6 +2304,9 @@ do_unlock_pp(struct thread *td, struct u if (error != 0) return (error); + if ((flags & UMUTEX_ROBUST) != 0) + update_robst(m, td_exit); + if (rceiling == -1) new_inherited_pri = PRI_MAX; else { @@ -2398,6 +2468,7 @@ do_lock_umutex(struct thread *td, struct struct timespec *timeout, int mode, int wflags) { struct timespec cts, ets, tts; + struct robust_info *rob = NULL; struct timeval tv; uint32_t flags; int error; @@ -2406,6 +2477,12 @@ do_lock_umutex(struct thread *td, struct if (flags == -1) return (EFAULT); + if ((flags & UMUTEX_ROBUST) != 0 && mode != _UMUTEX_WAIT) { + rob = robust_alloc(); + rob->ownertd = td; + rob->umtxp = m; + } + if (timeout == NULL) { error = _do_lock_umutex(td, m, flags, 0, mode); /* Mutex locking is restarted if it is interrupted. */ @@ -2444,6 +2521,31 @@ do_lock_umutex(struct thread *td, struct if (error == ERESTART) error = EINTR; } + + if (error == 0) { + if ((flags & UMUTEX_ROBUST) != 0 && mode != _UMUTEX_WAIT) { + uint32_t robst = fubyte(&m->m_robstate); + + if (robst == UMUTEX_ROBST_OWNERDEAD) { + subyte(&m->m_robstate, + UMUTEX_ROBST_INCONSISTENT); + error = EOWNERDEAD; + } else if (robst == UMUTEX_ROBST_INCONSISTENT) { + error = EOWNERDEAD; + } else if (robst == UMUTEX_ROBST_NOTRECOVERABLE) { + error = ENOTRECOVERABLE; + if (rob != NULL) + robust_free(rob); + do_unlock_umutex(td, m, 0); + goto out; + } + if (rob != NULL && robust_insert(td, rob)) + robust_free(rob); + } + } else if (rob != NULL) { + robust_free(rob); + } +out: return (error); } @@ -2451,9 +2553,13 @@ do_lock_umutex(struct thread *td, struct * Unlock a userland POSIX mutex. */ static int -do_unlock_umutex(struct thread *td, struct umutex *m) +do_unlock_umutex(struct thread *td, struct umutex *m, int td_exit) { uint32_t flags; + int error; + + if (td_exit) + robust_remove(td, m); flags = fuword32(&m->m_flags); if (flags == -1) @@ -2461,14 +2567,20 @@ do_unlock_umutex(struct thread *td, stru switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { case 0: - return (do_unlock_normal(td, m, flags)); + error = do_unlock_normal(td, m, flags, td_exit); + break; case UMUTEX_PRIO_INHERIT: - return (do_unlock_pi(td, m, flags)); + error = do_unlock_pi(td, m, flags, td_exit); + break; case UMUTEX_PRIO_PROTECT: - return (do_unlock_pp(td, m, flags)); + error = do_unlock_pp(td, m, flags, td_exit); + break; + default: + error = EINVAL; } - - return (EINVAL); + if ((flags & UMUTEX_ROBUST) != 0 && !td_exit) + robust_remove(td, m); + return (error); } static int @@ -2590,7 +2702,7 @@ ignore: umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); - error = do_unlock_umutex(td, m); + error = do_unlock_umutex(td, m, 0); if (error) { UMTX_STATE_INC(cv_unlock_failure); error = 0; /* ignore the error */ @@ -3601,7 +3713,7 @@ __umtx_op_wake_umutex(struct thread *td, static int __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap) { - return do_unlock_umutex(td, uap->obj); + return do_unlock_umutex(td, uap->obj, 0); } static int @@ -4027,6 +4139,79 @@ freebsd32_umtx_op(struct thread *td, str } #endif +struct robust_info * +robust_alloc(void) +{ + struct robust_info *rb; + + rb = uma_zalloc(robust_zone, M_ZERO|M_WAITOK); + return (rb); +} + +static void +robust_free(struct robust_info *rb) +{ + uma_zfree(robust_zone, rb); +} + +static unsigned int +robust_hash(struct umutex *m) +{ + unsigned n = (uintptr_t)m; + return ((n * GOLDEN_RATIO_PRIME) >> ROBUST_SHIFTS) % ROBUST_CHAINS; +} + +static int +robust_insert(struct thread *td, struct robust_info *rob) +{ + struct umtx_q *uq = td->td_umtxq; + struct robust_info *rob2; + int hash = robust_hash(rob->umtxp); + struct robust_chain *robc = &robust_chains[hash]; + + mtx_lock(&robc->lock); + SLIST_FOREACH(rob2, &robc->rob_list, hash_qe) { + if (rob2->ownertd == td && + rob2->umtxp == rob->umtxp) { + mtx_unlock(&robc->lock); + return (EEXIST); + } + } + rob->ownertd = td; + SLIST_INSERT_HEAD(&robc->rob_list, rob, hash_qe); + mtx_unlock(&robc->lock); + LIST_INSERT_HEAD(&uq->uq_rob_list, rob, td_qe); + return (0); +} + +static void +robust_remove(struct thread *td, struct umutex *umtxp) +{ + struct robust_info *rob, *rob2; + int hash = robust_hash(umtxp); + struct robust_chain *robc = &robust_chains[hash]; + + rob2 = NULL; + mtx_lock(&robc->lock); + SLIST_FOREACH(rob, &robc->rob_list, hash_qe) { + if (rob->ownertd == td && + rob->umtxp == umtxp) { + if (rob2 == NULL) { + SLIST_REMOVE_HEAD(&robc->rob_list, hash_qe); + } else { + SLIST_REMOVE_AFTER(rob2, hash_qe); + } + break; + } + rob2 = rob; + } + mtx_unlock(&robc->lock); + if (rob != NULL) { + LIST_REMOVE(rob, td_qe); + robust_free(rob); + } +} + void umtx_thread_init(struct thread *td) { @@ -4058,7 +4243,7 @@ umtx_thread_alloc(struct thread *td) } /* - * exec() hook. + * exec() hook, clean up lastest thread's umtx info. */ static void umtx_exec_hook(void *arg __unused, struct proc *p __unused, @@ -4068,6 +4253,15 @@ umtx_exec_hook(void *arg __unused, struc } /* + * exit1() hook, clean up lastest thread's umtx info. + */ +static void +umtx_exit_hook(void *arg __unused, struct proc *p __unused) +{ + umtx_thread_cleanup(curthread); +} + +/* * thread_exit() hook. */ void @@ -4084,10 +4278,14 @@ umtx_thread_cleanup(struct thread *td) { struct umtx_q *uq; struct umtx_pi *pi; + struct robust_info *rob; if ((uq = td->td_umtxq) == NULL) return; + while ((rob = LIST_FIRST(&uq->uq_rob_list)) != NULL) + do_unlock_umutex(td, rob->umtxp, 1); + mtx_lock_spin(&umtx_lock); uq->uq_inherited_pri = PRI_MAX; while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) { Modified: user/davidxu/libthr/sys/sys/_umtx.h ============================================================================== --- user/davidxu/libthr/sys/sys/_umtx.h Sun Nov 7 12:29:26 2010 (r214913) +++ user/davidxu/libthr/sys/sys/_umtx.h Sun Nov 7 13:41:17 2010 (r214914) @@ -40,7 +40,9 @@ struct umutex { volatile __lwpid_t m_owner; /* Owner of the mutex */ __uint32_t m_flags; /* Flags of the mutex */ __uint32_t m_ceilings[2]; /* Priority protect ceiling */ - __uint32_t m_spare[4]; + __uint8_t m_robstate; + __uint8_t m_spare1[3]; + __uint32_t m_spare2[3]; }; struct ucond { Modified: user/davidxu/libthr/sys/sys/umtx.h ============================================================================== --- user/davidxu/libthr/sys/sys/umtx.h Sun Nov 7 12:29:26 2010 (r214913) +++ user/davidxu/libthr/sys/sys/umtx.h Sun Nov 7 13:41:17 2010 (r214914) @@ -45,20 +45,26 @@ #define UMUTEX_ERROR_CHECK 0x0002 /* Error-checking mutex */ #define UMUTEX_PRIO_INHERIT 0x0004 /* Priority inherited mutex */ #define UMUTEX_PRIO_PROTECT 0x0008 /* Priority protect mutex */ -#define UMUTEX_SIMPLE 0x0010 /* Use simple lock id. */ +#define UMUTEX_SIMPLE 0x0010 /* Use simple lock id. */ +#define UMUTEX_ROBUST 0x0020 -#define UMUTEX_SIMPLE_OWNER 1 /* The simple mutex's lock bit. */ +#define UMUTEX_SIMPLE_OWNER 1 /* The simple mutex's lock bit. */ + +#define UMUTEX_ROBST_NORMAL 0 +#define UMUTEX_ROBST_OWNERDEAD 1 +#define UMUTEX_ROBST_INCONSISTENT 2 +#define UMUTEX_ROBST_NOTRECOVERABLE 3 /* urwlock flags */ -#define URWLOCK_PREFER_READER 0x0002 +#define URWLOCK_PREFER_READER 0x0002 -#define URWLOCK_WRITE_OWNER 0x80000000U -#define URWLOCK_WRITE_WAITERS 0x40000000U -#define URWLOCK_READ_WAITERS 0x20000000U -#define URWLOCK_MAX_READERS 0x1fffffffU -#define URWLOCK_READER_COUNT(c) ((c) & URWLOCK_MAX_READERS) +#define URWLOCK_WRITE_OWNER 0x80000000U +#define URWLOCK_WRITE_WAITERS 0x40000000U +#define URWLOCK_READ_WAITERS 0x20000000U +#define URWLOCK_MAX_READERS 0x1fffffffU +#define URWLOCK_READER_COUNT(c) ((c) & URWLOCK_MAX_READERS) -#define UCOND_BIND_MUTEX 0x0002 +#define UCOND_BIND_MUTEX 0x0002 /* _usem flags */ #define SEM_NAMED 0x0002
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201011071341.oA7DfILV048093>