Date: Thu, 5 Oct 2017 19:18:02 +0000 (UTC) From: Mateusz Guzik <mjg@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r324335 - in head/sys: kern sys Message-ID: <201710051918.v95JI2aA016034@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: mjg Date: Thu Oct 5 19:18:02 2017 New Revision: 324335 URL: https://svnweb.freebsd.org/changeset/base/324335 Log: locks: take the number of readers into account when waiting Previous code would always spin once before checking the lock. But a lock with e.g. 6 readers is not going to become free in the duration of once spin even if they start draining immediately. Conservatively perform one for each reader. Note that the total number of allowed spins is still extremely small and is subject to change later. MFC after: 1 week Modified: head/sys/kern/kern_rwlock.c head/sys/kern/kern_sx.c head/sys/sys/lock.h Modified: head/sys/kern/kern_rwlock.c ============================================================================== --- head/sys/kern/kern_rwlock.c Thu Oct 5 19:11:25 2017 (r324334) +++ head/sys/kern/kern_rwlock.c Thu Oct 5 19:18:02 2017 (r324335) @@ -414,7 +414,7 @@ __rw_rlock_hard(volatile uintptr_t *c, struct thread * #ifdef ADAPTIVE_RWLOCKS volatile struct thread *owner; int spintries = 0; - int i; + int i, n; #endif #ifdef LOCK_PROFILING uint64_t waittime = 0; @@ -488,8 +488,9 @@ __rw_rlock_hard(volatile uintptr_t *c, struct thread * KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread), "spinning", "lockname:\"%s\"", rw->lock_object.lo_name); - for (i = 0; i < rowner_loops; i++) { - cpu_spinwait(); + for (i = 0; i < rowner_loops; i += n) { + n = RW_READERS(v); + lock_delay_spin(n); v = RW_READ_VALUE(rw); if ((v & RW_LOCK_READ) == 0 || RW_CAN_READ(td, v)) break; @@ -830,7 +831,7 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, ui #ifdef ADAPTIVE_RWLOCKS volatile struct thread *owner; int spintries = 0; - int i; + int i, n; #endif uintptr_t x; #ifdef LOCK_PROFILING @@ -928,8 +929,9 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, ui KTR_STATE1(KTR_SCHED, "thread", sched_tdname(curthread), "spinning", "lockname:\"%s\"", rw->lock_object.lo_name); - for (i = 0; i < rowner_loops; i++) { - cpu_spinwait(); + for (i = 0; i < rowner_loops; i += n) { + n = RW_READERS(v); + lock_delay_spin(n); v = RW_READ_VALUE(rw); if ((v & RW_LOCK_WRITE_SPINNER) == 0) break; Modified: head/sys/kern/kern_sx.c ============================================================================== --- head/sys/kern/kern_sx.c Thu Oct 5 19:11:25 2017 (r324334) +++ head/sys/kern/kern_sx.c Thu Oct 5 19:18:02 2017 (r324335) @@ -502,7 +502,7 @@ _sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t t GIANT_DECLARE; #ifdef ADAPTIVE_SX volatile struct thread *owner; - u_int i, spintries = 0; + u_int i, n, spintries = 0; #endif #ifdef LOCK_PROFILING uint64_t waittime = 0; @@ -600,12 +600,13 @@ _sx_xlock_hard(struct sx *sx, uintptr_t x, uintptr_t t "lockname:\"%s\"", sx->lock_object.lo_name); GIANT_SAVE(); spintries++; - for (i = 0; i < asx_loops; i++) { + for (i = 0; i < asx_loops; i += n) { if (LOCK_LOG_TEST(&sx->lock_object, 0)) CTR4(KTR_LOCK, "%s: shared spinning on %p with %u and %u", __func__, sx, spintries, i); - cpu_spinwait(); + n = SX_SHARERS(x); + lock_delay_spin(n); x = SX_READ_VALUE(sx); if ((x & SX_LOCK_SHARED) == 0 || SX_SHARERS(x) == 0) Modified: head/sys/sys/lock.h ============================================================================== --- head/sys/sys/lock.h Thu Oct 5 19:11:25 2017 (r324334) +++ head/sys/sys/lock.h Thu Oct 5 19:18:02 2017 (r324335) @@ -226,6 +226,13 @@ lock_delay_arg_init(struct lock_delay_arg *la, struct la->spin_cnt = 0; } +#define lock_delay_spin(n) do { \ + u_int _i; \ + \ + for (_i = (n); _i > 0; _i--) \ + cpu_spinwait(); \ +} while (0) + #define LOCK_DELAY_SYSINIT(func) \ SYSINIT(func##_ld, SI_SUB_LOCK, SI_ORDER_ANY, func, NULL)
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201710051918.v95JI2aA016034>