Date: Tue, 28 Oct 2014 15:30:34 +0000 (UTC) From: Konstantin Belousov <kib@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r273785 - head/sys/kern Message-ID: <201410281530.s9SFUYBd015691@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kib Date: Tue Oct 28 15:30:33 2014 New Revision: 273785 URL: https://svnweb.freebsd.org/changeset/base/273785 Log: Convert kern_umtx.c to use fueword() and casueword(). Also fix some mishandling of suword(9) errors as errno, which resulted in spurious ERESTART. Sponsored by: The FreeBSD Foundation Tested by: pho MFC after: 3 weeks Modified: head/sys/kern/kern_umtx.c Modified: head/sys/kern/kern_umtx.c ============================================================================== --- head/sys/kern/kern_umtx.c Tue Oct 28 15:28:20 2014 (r273784) +++ head/sys/kern/kern_umtx.c Tue Oct 28 15:30:33 2014 (r273785) @@ -510,6 +510,15 @@ umtxq_unbusy(struct umtx_key *key) wakeup_one(uc); } +static inline void +umtxq_unbusy_unlocked(struct umtx_key *key) +{ + + umtxq_lock(key); + umtxq_unbusy(key); + umtxq_unlock(key); +} + static struct umtxq_queue * umtxq_queue_lookup(struct umtx_key *key, int q) { @@ -847,6 +856,7 @@ do_wait(struct thread *td, void *addr, u struct abs_timeout timo; struct umtx_q *uq; u_long tmp; + uint32_t tmp32; int error = 0; uq = td->td_umtxq; @@ -860,18 +870,29 @@ do_wait(struct thread *td, void *addr, u umtxq_lock(&uq->uq_key); umtxq_insert(uq); umtxq_unlock(&uq->uq_key); - if (compat32 == 0) - tmp = fuword(addr); - else - tmp = (unsigned int)fuword32(addr); + if (compat32 == 0) { + error = fueword(addr, &tmp); + if (error != 0) + error = EFAULT; + } else { + error = fueword32(addr, &tmp32); + if (error == 0) + tmp = tmp32; + else + error = EFAULT; + } umtxq_lock(&uq->uq_key); - if (tmp == id) - error = umtxq_sleep(uq, "uwait", timeout == NULL ? - NULL : &timo); - if ((uq->uq_flags & UQF_UMTXQ) == 0) - error = 0; - else + if (error == 0) { + if (tmp == id) + error = umtxq_sleep(uq, "uwait", timeout == NULL ? + NULL : &timo); + if ((uq->uq_flags & UQF_UMTXQ) == 0) + error = 0; + else + umtxq_remove(uq); + } else if ((uq->uq_flags & UQF_UMTXQ) != 0) { umtxq_remove(uq); + } umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); if (error == ERESTART) @@ -908,11 +929,11 @@ do_lock_normal(struct thread *td, struct struct abs_timeout timo; struct umtx_q *uq; uint32_t owner, old, id; - int error = 0; + int error, rv; id = td->td_tid; uq = td->td_umtxq; - + error = 0; if (timeout != NULL) abs_timeout_init2(&timo, timeout); @@ -921,7 +942,9 @@ do_lock_normal(struct thread *td, struct * can fault on any access. */ for (;;) { - owner = fuword32(__DEVOLATILE(void *, &m->m_owner)); + rv = fueword32(__DEVOLATILE(void *, &m->m_owner), &owner); + if (rv == -1) + return (EFAULT); if (mode == _UMUTEX_WAIT) { if (owner == UMUTEX_UNOWNED || owner == UMUTEX_CONTESTED) return (0); @@ -929,31 +952,31 @@ do_lock_normal(struct thread *td, struct /* * Try the uncontested case. This should be done in userland. */ - owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id); + rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, + &owner, id); + /* The address was invalid. */ + if (rv == -1) + return (EFAULT); /* The acquire succeeded. */ if (owner == UMUTEX_UNOWNED) return (0); - /* The address was invalid. */ - if (owner == -1) - return (EFAULT); - /* If no one owns it but it is contested try to acquire it. */ if (owner == UMUTEX_CONTESTED) { - owner = casuword32(&m->m_owner, - UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); + rv = casueword32(&m->m_owner, + UMUTEX_CONTESTED, &owner, + id | UMUTEX_CONTESTED); + /* The address was invalid. */ + if (rv == -1) + return (EFAULT); if (owner == UMUTEX_CONTESTED) return (0); - /* The address was invalid. */ - if (owner == -1) - return (EFAULT); - - error = umtxq_check_susp(td); - if (error != 0) - return (error); + rv = umtxq_check_susp(td); + if (rv != 0) + return (rv); /* If this failed the lock has changed, restart. */ continue; @@ -985,10 +1008,11 @@ do_lock_normal(struct thread *td, struct * either some one else has acquired the lock or it has been * released. */ - old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED); + rv = casueword32(&m->m_owner, owner, &old, + owner | UMUTEX_CONTESTED); /* The address was invalid. */ - if (old == -1) { + if (rv == -1) { umtxq_lock(&uq->uq_key); umtxq_remove(uq); umtxq_unbusy(&uq->uq_key); @@ -1033,16 +1057,16 @@ do_unlock_normal(struct thread *td, stru /* * Make sure we own this mtx. */ - owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); - if (owner == -1) + error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), &owner); + if (error == -1) return (EFAULT); if ((owner & ~UMUTEX_CONTESTED) != id) return (EPERM); if ((owner & UMUTEX_CONTESTED) == 0) { - old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED); - if (old == -1) + error = casueword32(&m->m_owner, owner, &old, UMUTEX_UNOWNED); + if (error == -1) return (EFAULT); if (old == owner) return (0); @@ -1064,14 +1088,14 @@ do_unlock_normal(struct thread *td, stru * there is zero or one thread only waiting for it. * Otherwise, it must be marked as contested. */ - old = casuword32(&m->m_owner, owner, - count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); + error = casueword32(&m->m_owner, owner, &old, + count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); umtxq_lock(&key); umtxq_signal(&key,1); umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); - if (old == -1) + if (error == -1) return (EFAULT); if (old != owner) return (EINVAL); @@ -1091,14 +1115,16 @@ do_wake_umutex(struct thread *td, struct int error; int count; - owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); - if (owner == -1) + error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), &owner); + if (error == -1) return (EFAULT); if ((owner & ~UMUTEX_CONTESTED) != 0) return (0); - flags = fuword32(&m->m_flags); + error = fueword32(&m->m_flags, &flags); + if (error == -1) + return (EFAULT); /* We should only ever be in here for contested locks */ if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), @@ -1110,16 +1136,20 @@ do_wake_umutex(struct thread *td, struct count = umtxq_count(&key); umtxq_unlock(&key); - if (count <= 1) - owner = casuword32(&m->m_owner, UMUTEX_CONTESTED, UMUTEX_UNOWNED); + if (count <= 1) { + error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, + UMUTEX_UNOWNED); + if (error == -1) + error = EFAULT; + } umtxq_lock(&key); - if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0) + if (error == 0 && count != 0 && (owner & ~UMUTEX_CONTESTED) == 0) umtxq_signal(&key, 1); umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); - return (0); + return (error); } /* @@ -1162,41 +1192,49 @@ do_wake2_umutex(struct thread *td, struc * any memory. */ if (count > 1) { - owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); - while ((owner & UMUTEX_CONTESTED) ==0) { - old = casuword32(&m->m_owner, owner, - owner|UMUTEX_CONTESTED); + error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), + &owner); + if (error == -1) + error = EFAULT; + while (error == 0 && (owner & UMUTEX_CONTESTED) == 0) { + error = casueword32(&m->m_owner, owner, &old, + owner | UMUTEX_CONTESTED); + if (error == -1) { + error = EFAULT; + break; + } if (old == owner) break; owner = old; - if (old == -1) - break; error = umtxq_check_susp(td); if (error != 0) break; } } else if (count == 1) { - owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); - while ((owner & ~UMUTEX_CONTESTED) != 0 && + error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), + &owner); + if (error == -1) + error = EFAULT; + while (error == 0 && (owner & ~UMUTEX_CONTESTED) != 0 && (owner & UMUTEX_CONTESTED) == 0) { - old = casuword32(&m->m_owner, owner, - owner|UMUTEX_CONTESTED); + error = casueword32(&m->m_owner, owner, &old, + owner | UMUTEX_CONTESTED); + if (error == -1) { + error = EFAULT; + break; + } if (old == owner) break; owner = old; - if (old == -1) - break; error = umtxq_check_susp(td); if (error != 0) break; } } umtxq_lock(&key); - if (owner == -1) { - error = EFAULT; + if (error == EFAULT) { umtxq_signal(&key, INT_MAX); - } - else if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0) + } else if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0) umtxq_signal(&key, 1); umtxq_unbusy(&key); umtxq_unlock(&key); @@ -1576,7 +1614,7 @@ do_lock_pi(struct thread *td, struct umu struct umtx_q *uq; struct umtx_pi *pi, *new_pi; uint32_t id, owner, old; - int error; + int error, rv; id = td->td_tid; uq = td->td_umtxq; @@ -1619,7 +1657,12 @@ do_lock_pi(struct thread *td, struct umu /* * Try the uncontested case. This should be done in userland. */ - owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id); + rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id); + /* The address was invalid. */ + if (rv == -1) { + error = EFAULT; + break; + } /* The acquire succeeded. */ if (owner == UMUTEX_UNOWNED) { @@ -1627,16 +1670,15 @@ do_lock_pi(struct thread *td, struct umu break; } - /* The address was invalid. */ - if (owner == -1) { - error = EFAULT; - break; - } - /* If no one owns it but it is contested try to acquire it. */ if (owner == UMUTEX_CONTESTED) { - owner = casuword32(&m->m_owner, - UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); + rv = casueword32(&m->m_owner, + UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED); + /* The address was invalid. */ + if (rv == -1) { + error = EFAULT; + break; + } if (owner == UMUTEX_CONTESTED) { umtxq_lock(&uq->uq_key); @@ -1647,12 +1689,6 @@ do_lock_pi(struct thread *td, struct umu break; } - /* The address was invalid. */ - if (owner == -1) { - error = EFAULT; - break; - } - error = umtxq_check_susp(td); if (error != 0) break; @@ -1683,13 +1719,12 @@ do_lock_pi(struct thread *td, struct umu * either some one else has acquired the lock or it has been * released. */ - old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED); + rv = casueword32(&m->m_owner, owner, &old, + owner | UMUTEX_CONTESTED); /* The address was invalid. */ - if (old == -1) { - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + if (rv == -1) { + umtxq_unbusy_unlocked(&uq->uq_key); error = EFAULT; break; } @@ -1741,8 +1776,8 @@ do_unlock_pi(struct thread *td, struct u /* * Make sure we own this mtx. */ - owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); - if (owner == -1) + error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), &owner); + if (error == -1) return (EFAULT); if ((owner & ~UMUTEX_CONTESTED) != id) @@ -1750,8 +1785,8 @@ do_unlock_pi(struct thread *td, struct u /* This should be done in userland */ if ((owner & UMUTEX_CONTESTED) == 0) { - old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED); - if (old == -1) + error = casueword32(&m->m_owner, owner, &old, UMUTEX_UNOWNED); + if (error == -1) return (EFAULT); if (old == owner) return (0); @@ -1809,14 +1844,12 @@ do_unlock_pi(struct thread *td, struct u * there is zero or one thread only waiting for it. * Otherwise, it must be marked as contested. */ - old = casuword32(&m->m_owner, owner, - count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); + error = casueword32(&m->m_owner, owner, &old, + count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); - umtxq_lock(&key); - umtxq_unbusy(&key); - umtxq_unlock(&key); + umtxq_unbusy_unlocked(&key); umtx_key_release(&key); - if (old == -1) + if (error == -1) return (EFAULT); if (old != owner) return (EINVAL); @@ -1835,7 +1868,7 @@ do_lock_pp(struct thread *td, struct umu struct umtx_pi *pi; uint32_t ceiling; uint32_t owner, id; - int error, pri, old_inherited_pri, su; + int error, pri, old_inherited_pri, su, rv; id = td->td_tid; uq = td->td_umtxq; @@ -1853,7 +1886,12 @@ do_lock_pp(struct thread *td, struct umu umtxq_busy(&uq->uq_key); umtxq_unlock(&uq->uq_key); - ceiling = RTP_PRIO_MAX - fuword32(&m->m_ceilings[0]); + rv = fueword32(&m->m_ceilings[0], &ceiling); + if (rv == -1) { + error = EFAULT; + goto out; + } + ceiling = RTP_PRIO_MAX - ceiling; if (ceiling > RTP_PRIO_MAX) { error = EINVAL; goto out; @@ -1874,17 +1912,16 @@ do_lock_pp(struct thread *td, struct umu } mtx_unlock_spin(&umtx_lock); - owner = casuword32(&m->m_owner, - UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); - - if (owner == UMUTEX_CONTESTED) { - error = 0; + rv = casueword32(&m->m_owner, + UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED); + /* The address was invalid. */ + if (rv == -1) { + error = EFAULT; break; } - /* The address was invalid. */ - if (owner == -1) { - error = EFAULT; + if (owner == UMUTEX_CONTESTED) { + error = 0; break; } @@ -1946,9 +1983,7 @@ do_lock_pp(struct thread *td, struct umu } out: - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_unbusy_unlocked(&uq->uq_key); umtx_key_release(&uq->uq_key); return (error); } @@ -1973,8 +2008,8 @@ do_unlock_pp(struct thread *td, struct u /* * Make sure we own this mtx. */ - owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); - if (owner == -1) + error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), &owner); + if (error == -1) return (EFAULT); if ((owner & ~UMUTEX_CONTESTED) != id) @@ -2047,9 +2082,11 @@ do_set_ceiling(struct thread *td, struct uint32_t save_ceiling; uint32_t owner, id; uint32_t flags; - int error; + int error, rv; - flags = fuword32(&m->m_flags); + error = fueword32(&m->m_flags, &flags); + if (error == -1) + return (EFAULT); if ((flags & UMUTEX_PRIO_PROTECT) == 0) return (EINVAL); if (ceiling > RTP_PRIO_MAX) @@ -2064,10 +2101,18 @@ do_set_ceiling(struct thread *td, struct umtxq_busy(&uq->uq_key); umtxq_unlock(&uq->uq_key); - save_ceiling = fuword32(&m->m_ceilings[0]); + rv = fueword32(&m->m_ceilings[0], &save_ceiling); + if (rv == -1) { + error = EFAULT; + break; + } - owner = casuword32(&m->m_owner, - UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); + rv = casueword32(&m->m_owner, + UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED); + if (rv == -1) { + error = EFAULT; + break; + } if (owner == UMUTEX_CONTESTED) { suword32(&m->m_ceilings[0], ceiling); @@ -2077,12 +2122,6 @@ do_set_ceiling(struct thread *td, struct break; } - /* The address was invalid. */ - if (owner == -1) { - error = EFAULT; - break; - } - if ((owner & ~UMUTEX_CONTESTED) == id) { suword32(&m->m_ceilings[0], ceiling); error = 0; @@ -2129,8 +2168,8 @@ do_lock_umutex(struct thread *td, struct uint32_t flags; int error; - flags = fuword32(&m->m_flags); - if (flags == -1) + error = fueword32(&m->m_flags, &flags); + if (error == -1) return (EFAULT); switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { @@ -2164,9 +2203,10 @@ static int do_unlock_umutex(struct thread *td, struct umutex *m) { uint32_t flags; + int error; - flags = fuword32(&m->m_flags); - if (flags == -1) + error = fueword32(&m->m_flags, &flags); + if (error == -1) return (EFAULT); switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { @@ -2187,21 +2227,27 @@ do_cv_wait(struct thread *td, struct uco { struct abs_timeout timo; struct umtx_q *uq; - uint32_t flags; - uint32_t clockid; + uint32_t flags, clockid, hasw; int error; uq = td->td_umtxq; - flags = fuword32(&cv->c_flags); + error = fueword32(&cv->c_flags, &flags); + if (error == -1) + return (EFAULT); error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key); if (error != 0) return (error); if ((wflags & CVWAIT_CLOCKID) != 0) { - clockid = fuword32(&cv->c_clockid); + error = fueword32(&cv->c_clockid, &clockid); + if (error == -1) { + umtx_key_release(&uq->uq_key); + return (EFAULT); + } if (clockid < CLOCK_REALTIME || clockid >= CLOCK_THREAD_CPUTIME_ID) { /* hmm, only HW clock id will work. */ + umtx_key_release(&uq->uq_key); return (EINVAL); } } else { @@ -2217,12 +2263,12 @@ do_cv_wait(struct thread *td, struct uco * Set c_has_waiters to 1 before releasing user mutex, also * don't modify cache line when unnecessary. */ - if (fuword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters)) == 0) + error = fueword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), + &hasw); + if (error == 0 && hasw == 0) suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 1); - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_unbusy_unlocked(&uq->uq_key); error = do_unlock_umutex(td, m); @@ -2276,7 +2322,9 @@ do_cv_signal(struct thread *td, struct u int error, cnt, nwake; uint32_t flags; - flags = fuword32(&cv->c_flags); + error = fueword32(&cv->c_flags, &flags); + if (error == -1) + return (EFAULT); if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) return (error); umtxq_lock(&key); @@ -2287,6 +2335,8 @@ do_cv_signal(struct thread *td, struct u umtxq_unlock(&key); error = suword32( __DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0); + if (error == -1) + error = EFAULT; umtxq_lock(&key); } umtxq_unbusy(&key); @@ -2302,7 +2352,9 @@ do_cv_broadcast(struct thread *td, struc int error; uint32_t flags; - flags = fuword32(&cv->c_flags); + error = fueword32(&cv->c_flags, &flags); + if (error == -1) + return (EFAULT); if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) return (error); @@ -2312,10 +2364,10 @@ do_cv_broadcast(struct thread *td, struc umtxq_unlock(&key); error = suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0); + if (error == -1) + error = EFAULT; - umtxq_lock(&key); - umtxq_unbusy(&key); - umtxq_unlock(&key); + umtxq_unbusy_unlocked(&key); umtx_key_release(&key); return (error); @@ -2329,10 +2381,12 @@ do_rw_rdlock(struct thread *td, struct u uint32_t flags, wrflags; int32_t state, oldstate; int32_t blocked_readers; - int error; + int error, rv; uq = td->td_umtxq; - flags = fuword32(&rwlock->rw_flags); + error = fueword32(&rwlock->rw_flags, &flags); + if (error == -1) + return (EFAULT); error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); if (error != 0) return (error); @@ -2345,15 +2399,22 @@ do_rw_rdlock(struct thread *td, struct u wrflags |= URWLOCK_WRITE_WAITERS; for (;;) { - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); + rv = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state), + &state); + if (rv == -1) { + umtx_key_release(&uq->uq_key); + return (EFAULT); + } + /* try to lock it */ while (!(state & wrflags)) { if (__predict_false(URWLOCK_READER_COUNT(state) == URWLOCK_MAX_READERS)) { umtx_key_release(&uq->uq_key); return (EAGAIN); } - oldstate = casuword32(&rwlock->rw_state, state, state + 1); - if (oldstate == -1) { + rv = casueword32(&rwlock->rw_state, state, + &oldstate, state + 1); + if (rv == -1) { umtx_key_release(&uq->uq_key); return (EFAULT); } @@ -2379,12 +2440,17 @@ do_rw_rdlock(struct thread *td, struct u * re-read the state, in case it changed between the try-lock above * and the check below */ - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); + rv = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state), + &state); + if (rv == -1) + error = EFAULT; /* set read contention bit */ - while ((state & wrflags) && !(state & URWLOCK_READ_WAITERS)) { - oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_READ_WAITERS); - if (oldstate == -1) { + while (error == 0 && (state & wrflags) && + !(state & URWLOCK_READ_WAITERS)) { + rv = casueword32(&rwlock->rw_state, state, + &oldstate, state | URWLOCK_READ_WAITERS); + if (rv == -1) { error = EFAULT; break; } @@ -2396,17 +2462,13 @@ do_rw_rdlock(struct thread *td, struct u break; } if (error != 0) { - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_unbusy_unlocked(&uq->uq_key); break; } /* state is changed while setting flags, restart */ if (!(state & wrflags)) { - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_unbusy_unlocked(&uq->uq_key); error = umtxq_check_susp(td); if (error != 0) break; @@ -2415,7 +2477,13 @@ do_rw_rdlock(struct thread *td, struct u sleep: /* contention bit is set, before sleeping, increase read waiter count */ - blocked_readers = fuword32(&rwlock->rw_blocked_readers); + rv = fueword32(&rwlock->rw_blocked_readers, + &blocked_readers); + if (rv == -1) { + umtxq_unbusy_unlocked(&uq->uq_key); + error = EFAULT; + break; + } suword32(&rwlock->rw_blocked_readers, blocked_readers+1); while (state & wrflags) { @@ -2431,18 +2499,32 @@ sleep: umtxq_unlock(&uq->uq_key); if (error) break; - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); + rv = fueword32(__DEVOLATILE(int32_t *, + &rwlock->rw_state), &state); + if (rv == -1) { + error = EFAULT; + break; + } } /* decrease read waiter count, and may clear read contention bit */ - blocked_readers = fuword32(&rwlock->rw_blocked_readers); + rv = fueword32(&rwlock->rw_blocked_readers, + &blocked_readers); + if (rv == -1) { + umtxq_unbusy_unlocked(&uq->uq_key); + error = EFAULT; + break; + } suword32(&rwlock->rw_blocked_readers, blocked_readers-1); if (blocked_readers == 1) { - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); - for (;;) { - oldstate = casuword32(&rwlock->rw_state, state, - state & ~URWLOCK_READ_WAITERS); - if (oldstate == -1) { + rv = fueword32(__DEVOLATILE(int32_t *, + &rwlock->rw_state), &state); + if (rv == -1) + error = EFAULT; + while (error == 0) { + rv = casueword32(&rwlock->rw_state, state, + &oldstate, state & ~URWLOCK_READ_WAITERS); + if (rv == -1) { error = EFAULT; break; } @@ -2450,14 +2532,10 @@ sleep: break; state = oldstate; error = umtxq_check_susp(td); - if (error != 0) - break; } } - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_unbusy_unlocked(&uq->uq_key); if (error != 0) break; } @@ -2476,10 +2554,12 @@ do_rw_wrlock(struct thread *td, struct u int32_t state, oldstate; int32_t blocked_writers; int32_t blocked_readers; - int error; + int error, rv; uq = td->td_umtxq; - flags = fuword32(&rwlock->rw_flags); + error = fueword32(&rwlock->rw_flags, &flags); + if (error == -1) + return (EFAULT); error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); if (error != 0) return (error); @@ -2489,10 +2569,16 @@ do_rw_wrlock(struct thread *td, struct u blocked_readers = 0; for (;;) { - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); + rv = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state), + &state); + if (rv == -1) { + umtx_key_release(&uq->uq_key); + return (EFAULT); + } while (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) { - oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_OWNER); - if (oldstate == -1) { + rv = casueword32(&rwlock->rw_state, state, + &oldstate, state | URWLOCK_WRITE_OWNER); + if (rv == -1) { umtx_key_release(&uq->uq_key); return (EFAULT); } @@ -2528,12 +2614,17 @@ do_rw_wrlock(struct thread *td, struct u * re-read the state, in case it changed between the try-lock above * and the check below */ - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); + rv = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state), + &state); + if (rv == -1) + error = EFAULT; - while (((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) && - (state & URWLOCK_WRITE_WAITERS) == 0) { - oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_WAITERS); - if (oldstate == -1) { + while (error == 0 && ((state & URWLOCK_WRITE_OWNER) || + URWLOCK_READER_COUNT(state) != 0) && + (state & URWLOCK_WRITE_WAITERS) == 0) { + rv = casueword32(&rwlock->rw_state, state, + &oldstate, state | URWLOCK_WRITE_WAITERS); + if (rv == -1) { error = EFAULT; break; } @@ -2545,23 +2636,25 @@ do_rw_wrlock(struct thread *td, struct u break; } if (error != 0) { - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_unbusy_unlocked(&uq->uq_key); break; } if (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) { - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_unbusy_unlocked(&uq->uq_key); error = umtxq_check_susp(td); if (error != 0) break; continue; } sleep: - blocked_writers = fuword32(&rwlock->rw_blocked_writers); + rv = fueword32(&rwlock->rw_blocked_writers, + &blocked_writers); + if (rv == -1) { + umtxq_unbusy_unlocked(&uq->uq_key); + error = EFAULT; + break; + } suword32(&rwlock->rw_blocked_writers, blocked_writers+1); while ((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) { @@ -2577,17 +2670,34 @@ sleep: umtxq_unlock(&uq->uq_key); if (error) break; - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); + rv = fueword32(__DEVOLATILE(int32_t *, + &rwlock->rw_state), &state); + if (rv == -1) { + error = EFAULT; + break; + } } - blocked_writers = fuword32(&rwlock->rw_blocked_writers); + rv = fueword32(&rwlock->rw_blocked_writers, + &blocked_writers); + if (rv == -1) { + umtxq_unbusy_unlocked(&uq->uq_key); + error = EFAULT; + break; + } suword32(&rwlock->rw_blocked_writers, blocked_writers-1); if (blocked_writers == 1) { - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); + rv = fueword32(__DEVOLATILE(int32_t *, + &rwlock->rw_state), &state); + if (rv == -1) { + umtxq_unbusy_unlocked(&uq->uq_key); + error = EFAULT; + break; + } for (;;) { - oldstate = casuword32(&rwlock->rw_state, state, - state & ~URWLOCK_WRITE_WAITERS); - if (oldstate == -1) { + rv = casueword32(&rwlock->rw_state, state, + &oldstate, state & ~URWLOCK_WRITE_WAITERS); + if (rv == -1) { error = EFAULT; break; } @@ -2603,13 +2713,17 @@ sleep: if (error != 0) break; } - blocked_readers = fuword32(&rwlock->rw_blocked_readers); + rv = fueword32(&rwlock->rw_blocked_readers, + &blocked_readers); + if (rv == -1) { + umtxq_unbusy_unlocked(&uq->uq_key); + error = EFAULT; + break; + } } else blocked_readers = 0; - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_unbusy_unlocked(&uq->uq_key); } umtx_key_release(&uq->uq_key); @@ -2624,20 +2738,26 @@ do_rw_unlock(struct thread *td, struct u struct umtx_q *uq; uint32_t flags; int32_t state, oldstate; - int error, q, count; + int error, rv, q, count; uq = td->td_umtxq; - flags = fuword32(&rwlock->rw_flags); + error = fueword32(&rwlock->rw_flags, &flags); + if (error == -1) + return (EFAULT); error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); if (error != 0) return (error); - state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); + error = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state), &state); + if (error == -1) { + error = EFAULT; + goto out; + } if (state & URWLOCK_WRITE_OWNER) { for (;;) { - oldstate = casuword32(&rwlock->rw_state, state, - state & ~URWLOCK_WRITE_OWNER); - if (oldstate == -1) { + rv = casueword32(&rwlock->rw_state, state, + &oldstate, state & ~URWLOCK_WRITE_OWNER); + if (rv == -1) { error = EFAULT; goto out; } @@ -2655,9 +2775,9 @@ do_rw_unlock(struct thread *td, struct u } *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201410281530.s9SFUYBd015691>