Skip site navigation (1)Skip section navigation (2) 
Date:      Mon, 5 Mar 2018 00:30:07 +0000 (UTC)
From:      Mateusz Guzik <mjg@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r330432 - head/sys/kern
Message-ID:  <201803050030.w250U7gm076203@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help 
Author: mjg
Date: Mon Mar  5 00:30:07 2018
New Revision: 330432
URL: https://svnweb.freebsd.org/changeset/base/330432

Log:
  lockmgr: save on sleepq when cmpset fails

Modified:
  head/sys/kern/kern_lock.c

Modified: head/sys/kern/kern_lock.c
==============================================================================
--- head/sys/kern/kern_lock.c	Mon Mar  5 00:15:56 2018	(r330431)
+++ head/sys/kern/kern_lock.c	Mon Mar  5 00:30:07 2018	(r330432)
@@ -296,7 +296,7 @@ sleeplk(struct lock *lk, u_int flags, struct lock_obje
 static __inline int
 wakeupshlk(struct lock *lk, const char *file, int line)
 {
-	uintptr_t v, x;
+	uintptr_t v, x, orig_x;
 	u_int realexslp;
 	int queue, wakeup_swapper;
 
@@ -311,7 +311,9 @@ wakeupshlk(struct lock *lk, const char *file, int line
 		 * path in order to handle wakeups correctly.
 		 */
 		sleepq_lock(&lk->lock_object);
-		x = lk->lk_lock & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
+		orig_x = lk->lk_lock;
+retry_sleepq:
+		x = orig_x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
 		v = LK_UNLOCKED;
 
 		/*
@@ -360,11 +362,16 @@ wakeupshlk(struct lock *lk, const char *file, int line
 			queue = SQ_SHARED_QUEUE;
 		}
 
-		if (!atomic_cmpset_rel_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x,
-		    v)) {
+		if (lockmgr_sunlock_try(lk, &orig_x)) {
 			sleepq_release(&lk->lock_object);
-			continue;
+			break;
 		}
+
+		x |= LK_SHARERS_LOCK(1);
+		if (!atomic_fcmpset_rel_ptr(&lk->lk_lock, &x, v)) {
+			orig_x = x;
+			goto retry_sleepq;
+		}
 		LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
 		    __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
 		    "exclusive");
@@ -602,6 +609,7 @@ lockmgr_slock_hard(struct lock *lk, u_int flags, struc
 		 */
 		sleepq_lock(&lk->lock_object);
 		x = lk->lk_lock;
+retry_sleepq:
 
 		/*
 		 * if the lock can be acquired in shared mode, try
@@ -617,10 +625,9 @@ lockmgr_slock_hard(struct lock *lk, u_int flags, struc
 		 * loop back and retry.
 		 */
 		if ((x & LK_SHARED_WAITERS) == 0) {
-			if (!atomic_cmpset_acq_ptr(&lk->lk_lock, x,
+			if (!atomic_fcmpset_acq_ptr(&lk->lk_lock, &x,
 			    x | LK_SHARED_WAITERS)) {
-				sleepq_release(&lk->lock_object);
-				continue;
+				goto retry_sleepq;
 			}
 			LOCK_LOG2(lk, "%s: %p set shared waiters flag",
 			    __func__, lk);
@@ -755,6 +762,7 @@ lockmgr_xlock_hard(struct lock *lk, u_int flags, struc
 		 */
 		sleepq_lock(&lk->lock_object);
 		x = lk->lk_lock;
+retry_sleepq:
 
 		/*
 		 * if the lock has been released while we spun on
@@ -777,7 +785,7 @@ lockmgr_xlock_hard(struct lock *lk, u_int flags, struc
 		v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS);
 		if ((x & ~v) == LK_UNLOCKED) {
 			v &= ~LK_EXCLUSIVE_SPINNERS;
-			if (atomic_cmpset_acq_ptr(&lk->lk_lock, x,
+			if (atomic_fcmpset_acq_ptr(&lk->lk_lock, &x,
 			    tid | v)) {
 				sleepq_release(&lk->lock_object);
 				LOCK_LOG2(lk,
@@ -785,8 +793,7 @@ lockmgr_xlock_hard(struct lock *lk, u_int flags, struc
 				    __func__, lk);
 				break;
 			}
-			sleepq_release(&lk->lock_object);
-			continue;
+			goto retry_sleepq;
 		}
 
 		/*
@@ -794,10 +801,9 @@ lockmgr_xlock_hard(struct lock *lk, u_int flags, struc
 		 * fail, loop back and retry.
 		 */
 		if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
-			if (!atomic_cmpset_ptr(&lk->lk_lock, x,
+			if (!atomic_fcmpset_ptr(&lk->lk_lock, &x,
 			    x | LK_EXCLUSIVE_WAITERS)) {
-				sleepq_release(&lk->lock_object);
-				continue;
+				goto retry_sleepq;
 			}
 			LOCK_LOG2(lk, "%s: %p set excl waiters flag",
 			    __func__, lk);

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201803050030.w250U7gm076203>