Date: Sun, 22 May 2016 12:35:50 +0000 (UTC) From: Dmitry Chagin <dchagin@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r300413 - head/sys/compat/linux Message-ID: <201605221235.u4MCZolo073115@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: dchagin Date: Sun May 22 12:35:50 2016 New Revision: 300413 URL: https://svnweb.freebsd.org/changeset/base/300413 Log: Due to lack the priority propagation feature replace sx by mutex. WIth this commit NPTL tests are ends in 1 minute faster. MFC after: 1 week Modified: head/sys/compat/linux/linux_futex.c Modified: head/sys/compat/linux/linux_futex.c ============================================================================== --- head/sys/compat/linux/linux_futex.c Sun May 22 12:28:55 2016 (r300412) +++ head/sys/compat/linux/linux_futex.c Sun May 22 12:35:50 2016 (r300413) @@ -54,9 +54,10 @@ __KERNEL_RCSID(1, "$NetBSD: linux_futex. #include <sys/queue.h> #include <sys/sched.h> #include <sys/sdt.h> -#include <sys/sx.h> #include <sys/umtx.h> +#include <vm/vm_extern.h> + #ifdef COMPAT_LINUX32 #include <machine/../linux32/linux.h> #include <machine/../linux32/linux32_proto.h> @@ -196,7 +197,7 @@ struct waiting_proc { }; struct futex { - struct sx f_lck; + struct mtx f_lck; uint32_t *f_uaddr; /* user-supplied value, for debug */ struct umtx_key f_key; uint32_t f_refcount; @@ -207,20 +208,22 @@ struct futex { struct futex_list futex_list; -#define FUTEX_LOCK(f) sx_xlock(&(f)->f_lck) -#define FUTEX_UNLOCK(f) sx_xunlock(&(f)->f_lck) +#define FUTEX_LOCK(f) mtx_lock(&(f)->f_lck) +#define FUTEX_LOCKED(f) mtx_owned(&(f)->f_lck) +#define FUTEX_UNLOCK(f) mtx_unlock(&(f)->f_lck) #define FUTEX_INIT(f) do { \ - sx_init_flags(&(f)->f_lck, "ftlk", \ - SX_DUPOK); \ + mtx_init(&(f)->f_lck, "ftlk", NULL, \ + MTX_DUPOK); \ LIN_SDT_PROBE1(futex, futex, create, \ &(f)->f_lck); \ } while (0) #define FUTEX_DESTROY(f) do { \ LIN_SDT_PROBE1(futex, futex, destroy, \ &(f)->f_lck); \ - sx_destroy(&(f)->f_lck); \ + mtx_destroy(&(f)->f_lck); \ } while (0) -#define FUTEX_ASSERT_LOCKED(f) sx_assert(&(f)->f_lck, SA_XLOCKED) +#define FUTEX_ASSERT_LOCKED(f) mtx_assert(&(f)->f_lck, MA_OWNED) +#define FUTEX_ASSERT_UNLOCKED(f) mtx_assert(&(f)->f_lck, MA_NOTOWNED) struct mtx futex_mtx; /* protects the futex list */ #define FUTEXES_LOCK do { \ @@ -239,6 +242,7 @@ struct mtx futex_mtx; /* protects the #define FUTEX_DONTCREATE 0x2 /* don't create futex if not exists */ #define FUTEX_DONTEXISTS 0x4 /* return EINVAL if futex exists */ #define FUTEX_SHARED 0x8 /* shared futex */ +#define FUTEX_DONTLOCK 0x10 /* don't lock futex */ /* wp_flags */ #define FUTEX_WP_REQUEUED 0x1 /* wp requeued - wp moved from wp_list @@ -258,6 +262,8 @@ static int futex_wake(struct futex *, in static int futex_requeue(struct futex *, int, struct futex *, int); static int futex_wait(struct futex *, struct waiting_proc *, int, uint32_t); +static void futex_lock(struct futex *); +static void futex_unlock(struct futex *); static int futex_atomic_op(struct thread *, int, uint32_t *); static int handle_futex_death(struct linux_emuldata *, uint32_t *, unsigned int); @@ -277,7 +283,6 @@ futex_put(struct futex *f, struct waitin { LIN_SDT_PROBE2(futex, futex_put, entry, f, wp); - FUTEX_ASSERT_LOCKED(f); if (wp != NULL) { if ((wp->wp_flags & FUTEX_WP_REMOVED) == 0) TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list); @@ -288,7 +293,8 @@ futex_put(struct futex *f, struct waitin if (--f->f_refcount == 0) { LIST_REMOVE(f, f_list); FUTEXES_UNLOCK; - FUTEX_UNLOCK(f); + if (FUTEX_LOCKED(f)) + futex_unlock(f); LIN_SDT_PROBE3(futex, futex_put, destroy, f->f_uaddr, f->f_refcount, f->f_key.shared); @@ -307,7 +313,8 @@ futex_put(struct futex *f, struct waitin LINUX_CTR3(sys_futex, "futex_put uaddr %p ref %d shared %d", f->f_uaddr, f->f_refcount, f->f_key.shared); FUTEXES_UNLOCK; - FUTEX_UNLOCK(f); + if (FUTEX_LOCKED(f)) + futex_unlock(f); LIN_SDT_PROBE0(futex, futex_put, return); } @@ -335,7 +342,8 @@ retry: LIST_FOREACH(f, &futex_list, f_list) { if (umtx_key_match(&f->f_key, &key)) { if (tmpf != NULL) { - FUTEX_UNLOCK(tmpf); + if (FUTEX_LOCKED(tmpf)) + futex_unlock(tmpf); FUTEX_DESTROY(tmpf); free(tmpf, M_FUTEX); } @@ -356,7 +364,8 @@ retry: FUTEXES_UNLOCK; umtx_key_release(&key); - FUTEX_LOCK(f); + if ((flags & FUTEX_DONTLOCK) == 0) + futex_lock(f); *newf = f; LIN_SDT_PROBE3(futex, futex_get0, shared, uaddr, f->f_refcount, f->f_key.shared); @@ -392,7 +401,8 @@ retry: * Lock the new futex before an insert into the futex_list * to prevent futex usage by other. */ - FUTEX_LOCK(tmpf); + if ((flags & FUTEX_DONTLOCK) == 0) + futex_lock(tmpf); goto retry; } @@ -440,6 +450,26 @@ futex_get(uint32_t *uaddr, struct waitin return (error); } +static inline void +futex_lock(struct futex *f) +{ + + LINUX_CTR3(sys_futex, "futex_lock uaddr %p ref %d shared %d", + f->f_uaddr, f->f_refcount, f->f_key.shared); + FUTEX_ASSERT_UNLOCKED(f); + FUTEX_LOCK(f); +} + +static inline void +futex_unlock(struct futex *f) +{ + + LINUX_CTR3(sys_futex, "futex_unlock uaddr %p ref %d shared %d", + f->f_uaddr, f->f_refcount, f->f_key.shared); + FUTEX_ASSERT_LOCKED(f); + FUTEX_UNLOCK(f); +} + static int futex_sleep(struct futex *f, struct waiting_proc *wp, int timeout) { @@ -449,7 +479,7 @@ futex_sleep(struct futex *f, struct wait LIN_SDT_PROBE3(futex, futex_sleep, entry, f, wp, timeout); LINUX_CTR4(sys_futex, "futex_sleep enter uaddr %p wp %p timo %d ref %d", f->f_uaddr, wp, timeout, f->f_refcount); - error = sx_sleep(wp, &f->f_lck, PCATCH, "futex", timeout); + error = mtx_sleep(wp, &f->f_lck, PCATCH, "futex", timeout); if (wp->wp_flags & FUTEX_WP_REQUEUED) { KASSERT(f != wp->wp_futex, ("futex != wp_futex")); @@ -465,7 +495,7 @@ futex_sleep(struct futex *f, struct wait wp->wp_futex->f_refcount); futex_put(f, NULL); f = wp->wp_futex; - FUTEX_LOCK(f); + futex_lock(f); } else { if (error) { LIN_SDT_PROBE3(futex, futex_sleep, sleep_error, error, @@ -676,7 +706,7 @@ linux_sys_futex(struct thread *td, struc struct timespec timeout; struct timeval utv, ctv; int timeout_hz; - int error; + int error, save; uint32_t flags, val; LIN_SDT_PROBE2(futex, linux_sys_futex, entry, td, args); @@ -749,6 +779,7 @@ linux_sys_futex(struct thread *td, struc } else timeout_hz = 0; +retry0: error = futex_get(args->uaddr, &wp, &f, flags | FUTEX_CREATE_WP); if (error) { @@ -756,14 +787,16 @@ linux_sys_futex(struct thread *td, struc return (error); } - error = copyin(args->uaddr, &val, sizeof(val)); + error = copyin_nofault(args->uaddr, &val, sizeof(val)); if (error) { + futex_put(f, wp); + error = copyin(args->uaddr, &val, sizeof(val)); + if (error == 0) + goto retry0; LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error, error); LINUX_CTR1(sys_futex, "WAIT copyin failed %d", error); - futex_put(f, wp); - LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); return (error); } @@ -831,7 +864,8 @@ linux_sys_futex(struct thread *td, struc return (EINVAL); } - error = futex_get(args->uaddr, NULL, &f, flags); +retry1: + error = futex_get(args->uaddr, NULL, &f, flags | FUTEX_DONTLOCK); if (error) { LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); return (error); @@ -845,22 +879,26 @@ linux_sys_futex(struct thread *td, struc * returned by FUTEX_CMP_REQUEUE. */ error = futex_get(args->uaddr2, NULL, &f2, - flags | FUTEX_DONTEXISTS); + flags | FUTEX_DONTEXISTS | FUTEX_DONTLOCK); if (error) { futex_put(f, NULL); LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); return (error); } - error = copyin(args->uaddr, &val, sizeof(val)); + futex_lock(f); + futex_lock(f2); + error = copyin_nofault(args->uaddr, &val, sizeof(val)); if (error) { + futex_put(f2, NULL); + futex_put(f, NULL); + error = copyin(args->uaddr, &val, sizeof(val)); + if (error == 0) + goto retry1; LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error, error); LINUX_CTR1(sys_futex, "CMP_REQUEUE copyin failed %d", error); - futex_put(f2, NULL); - futex_put(f, NULL); - LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); return (error); } @@ -890,50 +928,45 @@ linux_sys_futex(struct thread *td, struc args->uaddr, args->val, args->uaddr2, args->val3, args->timeout); - error = futex_get(args->uaddr, NULL, &f, flags); +retry2: + error = futex_get(args->uaddr, NULL, &f, flags | FUTEX_DONTLOCK); if (error) { LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); return (error); } if (args->uaddr != args->uaddr2) - error = futex_get(args->uaddr2, NULL, &f2, flags); + error = futex_get(args->uaddr2, NULL, &f2, + flags | FUTEX_DONTLOCK); if (error) { futex_put(f, NULL); LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); return (error); } + futex_lock(f); + futex_lock(f2); /* * This function returns positive number as results and * negative as errors */ + save = vm_fault_disable_pagefaults(); op_ret = futex_atomic_op(td, args->val3, args->uaddr2); + vm_fault_enable_pagefaults(save); LINUX_CTR2(sys_futex, "WAKE_OP atomic_op uaddr %p ret 0x%x", args->uaddr, op_ret); if (op_ret < 0) { - /* XXX: We don't handle the EFAULT yet. */ - if (op_ret != -EFAULT) { - if (f2 != NULL) - futex_put(f2, NULL); - futex_put(f, NULL); - - LIN_SDT_PROBE1(futex, linux_sys_futex, return, - -op_ret); - return (-op_ret); - } else { - LIN_SDT_PROBE0(futex, linux_sys_futex, - unhandled_efault); - } if (f2 != NULL) futex_put(f2, NULL); futex_put(f, NULL); - - LIN_SDT_PROBE1(futex, linux_sys_futex, return, EFAULT); - return (EFAULT); + error = copyin(args->uaddr2, &val, sizeof(val)); + if (error == 0) + goto retry2; + LIN_SDT_PROBE1(futex, linux_sys_futex, return, error); + return (error); } ret = futex_wake(f, args->val, args->val3);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201605221235.u4MCZolo073115>