Skip site navigation (1)Skip section navigation (2)
Date:      Mon, 9 Sep 2024 21:46:34 GMT
From:      Doug Moore <dougm@FreeBSD.org>
To:        src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org
Subject:   git: 942647057295 - main - swap_pager: clean up pctrie usage
Message-ID:  <202409092146.489LkYBL023066@gitrepo.freebsd.org>

next in thread | raw e-mail | index | archive | help
The branch main has been updated by dougm:

URL: https://cgit.FreeBSD.org/src/commit/?id=942647057295cd4720372dd85bcf6d1b52fbf38e

commit 942647057295cd4720372dd85bcf6d1b52fbf38e
Author:     Doug Moore <dougm@FreeBSD.org>
AuthorDate: 2024-09-09 21:44:13 +0000
Commit:     Doug Moore <dougm@FreeBSD.org>
CommitDate: 2024-09-09 21:44:13 +0000

    swap_pager: clean up pctrie usage
    
    Define wrapper functions for the pctrie operations specific to
    swap_pager, to hide some verbose details.  Separate the meta_transfer
    and meta_free functions into separate functions.
    
    Reviewed by:    kib (previous version)
    Tested by:      pho (previous version)
    Differential Revision:  https://reviews.freebsd.org/D46315
---
 sys/vm/swap_pager.c | 231 ++++++++++++++++++++++++++++++++--------------------
 1 file changed, 142 insertions(+), 89 deletions(-)

diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c
index 59d947c71279..c7a9f16a2953 100644
--- a/sys/vm/swap_pager.c
+++ b/sys/vm/swap_pager.c
@@ -491,7 +491,7 @@ static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t,
 static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t,
     vm_size_t *);
 static void swp_pager_meta_transfer(vm_object_t src, vm_object_t dst,
-    vm_pindex_t pindex, vm_pindex_t count, vm_size_t *freed);
+    vm_pindex_t pindex, vm_pindex_t count);
 static void swp_pager_meta_free_all(vm_object_t);
 static daddr_t swp_pager_meta_lookup(vm_object_t, vm_pindex_t);
 
@@ -531,6 +531,59 @@ swblk_trie_free(struct pctrie *ptree, void *node)
 
 PCTRIE_DEFINE(SWAP, swblk, p, swblk_trie_alloc, swblk_trie_free);
 
+static struct swblk *
+swblk_lookup(vm_object_t object, vm_pindex_t pindex)
+{
+	return (SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,
+	    rounddown(pindex, SWAP_META_PAGES)));
+}
+
+static struct swblk *
+swblk_start(vm_object_t object, vm_pindex_t pindex)
+{
+	return (SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,
+	    rounddown(pindex, SWAP_META_PAGES)));
+}
+
+static struct swblk *
+swblk_next(vm_object_t object, struct swblk *sb)
+{
+	return (swblk_start(object, sb->p + SWAP_META_PAGES));
+}
+
+static struct swblk *
+swblk_start_limit(vm_object_t object, vm_pindex_t pindex, vm_pindex_t limit)
+{
+	struct swblk *sb = swblk_start(object, pindex);
+	if (sb != NULL && sb->p < limit)
+		return (sb);
+	return (NULL);
+}
+
+static struct swblk *
+swblk_next_limit(vm_object_t object, struct swblk *sb, vm_pindex_t limit)
+{
+	return (swblk_start_limit(object, sb->p + SWAP_META_PAGES, limit));
+}
+
+static void
+swblk_lookup_remove(vm_object_t object, struct swblk *sb)
+{
+	SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p);
+}
+
+static int
+swblk_lookup_insert(vm_object_t object, struct swblk *sb)
+{
+	return (SWAP_PCTRIE_INSERT(&object->un_pager.swp.swp_blks, sb));
+}
+
+static bool
+swblk_is_empty(vm_object_t object)
+{
+	return (pctrie_is_empty(&object->un_pager.swp.swp_blks));
+}
+
 /*
  * SWP_SIZECHECK() -	update swap_pager_full indication
  *
@@ -1084,8 +1137,7 @@ swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
 	/*
 	 * Transfer source to destination.
 	 */
-	swp_pager_meta_transfer(srcobject, dstobject, offset, dstobject->size,
-	    NULL);
+	swp_pager_meta_transfer(srcobject, dstobject, offset, dstobject->size);
 
 	/*
 	 * Free left over swap blocks in source.
@@ -1218,8 +1270,7 @@ swap_pager_unswapped(vm_page_t m)
 	}
 	swap_pager_unswapped_acct(m);
 
-	sb = SWAP_PCTRIE_LOOKUP(&m->object->un_pager.swp.swp_blks,
-	    rounddown(m->pindex, SWAP_META_PAGES));
+	sb = swblk_lookup(m->object, m->pindex);
 	if (sb == NULL)
 		return;
 	range.start = sb->d[m->pindex % SWAP_META_PAGES];
@@ -1777,18 +1828,17 @@ u_long
 swap_pager_swapped_pages(vm_object_t object)
 {
 	struct swblk *sb;
-	vm_pindex_t pi;
 	u_long res;
 	int i;
 
 	VM_OBJECT_ASSERT_LOCKED(object);
 
-	if (pctrie_is_empty(&object->un_pager.swp.swp_blks))
+	if (swblk_is_empty(object))
 		return (0);
 
-	for (res = 0, pi = 0; (sb = SWAP_PCTRIE_LOOKUP_GE(
-	    &object->un_pager.swp.swp_blks, pi)) != NULL;
-	    pi = sb->p + SWAP_META_PAGES) {
+	res = 0;
+	for (sb = swblk_start(object, 0); sb != NULL;
+	    sb = swblk_next(object, sb)) {
 		for (i = 0; i < SWAP_META_PAGES; i++) {
 			if (sb->d[i] != SWAPBLK_NONE)
 				res++;
@@ -1835,16 +1885,14 @@ swap_pager_swapoff_object(struct swdevt *sp, vm_object_t object)
 		if (i == SWAP_META_PAGES) {
 			pi = sb->p + SWAP_META_PAGES;
 			if (sb_empty) {
-				SWAP_PCTRIE_REMOVE(
-				    &object->un_pager.swp.swp_blks, sb->p);
+				swblk_lookup_remove(object, sb);
 				uma_zfree(swblk_zone, sb);
 			}
 			i = 0;
 		}
 
 		if (i == 0) {
-			sb = SWAP_PCTRIE_LOOKUP_GE(
-			    &object->un_pager.swp.swp_blks, pi);
+			sb = swblk_start(object, pi);
 			if (sb == NULL)
 				break;
 			sb_empty = true;
@@ -2020,7 +2068,7 @@ swp_pager_free_empty_swblk(vm_object_t object, struct swblk *sb)
 {
 
 	if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
-		SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p);
+		swblk_lookup_remove(object, sb);
 		uma_zfree(swblk_zone, sb);
 	}
 }
@@ -2050,7 +2098,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk,
 	VM_OBJECT_ASSERT_WLOCKED(object);
 
 	rdpi = rounddown(pindex, SWAP_META_PAGES);
-	sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi);
+	sb = swblk_lookup(object, rdpi);
 	if (sb == NULL) {
 		if (swapblk == SWAPBLK_NONE)
 			return (SWAPBLK_NONE);
@@ -2079,8 +2127,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk,
 			} else
 				uma_zwait(swblk_zone);
 			VM_OBJECT_WLOCK(object);
-			sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,
-			    rdpi);
+			sb = swblk_lookup(object, rdpi);
 			if (sb != NULL)
 				/*
 				 * Somebody swapped out a nearby page,
@@ -2090,8 +2137,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk,
 				goto allocated;
 		}
 		for (;;) {
-			error = SWAP_PCTRIE_INSERT(
-			    &object->un_pager.swp.swp_blks, sb);
+			error = swblk_lookup_insert(object, sb);
 			if (error == 0) {
 				if (atomic_cmpset_int(&swpctrie_zone_exhausted,
 				    1, 0))
@@ -2113,8 +2159,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk,
 			} else
 				uma_zwait(swpctrie_zone);
 			VM_OBJECT_WLOCK(object);
-			sb1 = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,
-			    rdpi);
+			sb1 = swblk_lookup(object, rdpi);
 			if (sb1 != NULL) {
 				uma_zfree(swblk_zone, sb);
 				sb = sb1;
@@ -2142,53 +2187,42 @@ allocated:
 }
 
 /*
- * SWP_PAGER_META_TRANSFER() - free a range of blocks in the srcobject's swap
- * metadata, or transfer it into dstobject.
+ * SWP_PAGER_META_TRANSFER() - transfer a range of blocks in the srcobject's
+ * swap metadata into dstobject.
  *
  *	This routine will free swap metadata structures as they are cleaned
  *	out.
  */
 static void
 swp_pager_meta_transfer(vm_object_t srcobject, vm_object_t dstobject,
-    vm_pindex_t pindex, vm_pindex_t count, vm_size_t *moved)
+    vm_pindex_t pindex, vm_pindex_t count)
 {
 	struct page_range range;
 	struct swblk *sb;
 	daddr_t blk;
-	vm_page_t m;
 	vm_pindex_t offset, last;
-	vm_size_t mc;
 	int i, limit, start;
 
 	VM_OBJECT_ASSERT_WLOCKED(srcobject);
-	MPASS(moved == NULL || dstobject == NULL);
+	VM_OBJECT_ASSERT_WLOCKED(dstobject);
 
-	mc = 0;
-	m = NULL;
-	if (count == 0 || pctrie_is_empty(&srcobject->un_pager.swp.swp_blks))
-		goto out;
+	if (count == 0 || swblk_is_empty(srcobject))
+		return;
 
 	swp_pager_init_freerange(&range);
 	offset = pindex;
 	last = pindex + count;
-	for (;;) {
-		sb = SWAP_PCTRIE_LOOKUP_GE(&srcobject->un_pager.swp.swp_blks,
-		    rounddown(pindex, SWAP_META_PAGES));
-		if (sb == NULL || sb->p >= last)
-			break;
-		start = pindex > sb->p ? pindex - sb->p : 0;
-		limit = last - sb->p < SWAP_META_PAGES ? last - sb->p :
-		    SWAP_META_PAGES;
+	sb = swblk_start_limit(srcobject, pindex, last);
+	start = (sb != NULL && sb->p < pindex) ? pindex - sb->p : 0;
+	for (; sb != NULL;
+	    sb = swblk_start_limit(srcobject, pindex, last), start = 0) {
+		limit = MIN(last - sb->p, SWAP_META_PAGES);
 		for (i = start; i < limit; i++) {
-			blk = sb->d[i];
-			if (blk == SWAPBLK_NONE)
+			if (sb->d[i] == SWAPBLK_NONE)
 				continue;
-			if (dstobject == NULL ||
-			    (blk = swp_pager_meta_build(dstobject,
-			    sb->p + i - offset, blk, true),
-			    blk != sb->d[i] && blk != SWAPBLK_NONE))
-				swp_pager_update_freerange(&range, sb->d[i]);
-			else if (blk == sb->d[i]) {
+			blk = swp_pager_meta_build(dstobject,
+			    sb->p + i - offset, sb->d[i], true);
+			if (blk == sb->d[i]) {
 				/*
 				 * Destination has no swapblk and is not
 				 * resident, so transfer source.
@@ -2197,30 +2231,20 @@ swp_pager_meta_transfer(vm_object_t srcobject, vm_object_t dstobject,
 				 */
 				VM_OBJECT_WUNLOCK(srcobject);
 				swp_pager_meta_build(dstobject,
-				    sb->p + i - offset, blk, false);
+				    sb->p + i - offset, sb->d[i], false);
 				VM_OBJECT_WLOCK(srcobject);
-			}
-			if (moved != NULL) {
-				m = (m != NULL && m->pindex == sb->p + i - 1) ?
-				    vm_page_next(m) :
-				    vm_page_lookup(srcobject, sb->p + i);
-				if (m == NULL || vm_page_none_valid(m))
-					mc++;
-			}
+			} else if (blk != SWAPBLK_NONE)
+				swp_pager_update_freerange(&range, sb->d[i]);
 			sb->d[i] = SWAPBLK_NONE;
 		}
 		pindex = sb->p + SWAP_META_PAGES;
 		if (swp_pager_swblk_empty(sb, 0, start) &&
 		    swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {
-			SWAP_PCTRIE_REMOVE(&srcobject->un_pager.swp.swp_blks,
-			    sb->p);
+			swblk_lookup_remove(srcobject, sb);
 			uma_zfree(swblk_zone, sb);
 		}
 	}
 	swp_pager_freeswapspace(&range);
-out:
-	if (moved != NULL)
-		*moved = mc;
 }
 
 /*
@@ -2237,7 +2261,51 @@ static void
 swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count,
     vm_size_t *freed)
 {
-	swp_pager_meta_transfer(object, NULL, pindex, count, freed);
+	struct page_range range;
+	struct swblk *sb;
+	vm_page_t m;
+	vm_pindex_t last;
+	vm_size_t fc;
+	int i, limit, start;
+
+	VM_OBJECT_ASSERT_WLOCKED(object);
+
+	fc = 0;
+	m = NULL;
+	if (count == 0 || swblk_is_empty(object))
+		goto out;
+
+	swp_pager_init_freerange(&range);
+	last = pindex + count;
+	sb = swblk_start_limit(object, pindex, last);
+	start = (sb != NULL && sb->p < pindex) ? pindex - sb->p : 0;
+	for (; sb != NULL;
+	    sb = swblk_start_limit(object, pindex, last), start = 0) {
+		limit = MIN(last - sb->p, SWAP_META_PAGES);
+		for (i = start; i < limit; i++) {
+			if (sb->d[i] == SWAPBLK_NONE)
+				continue;
+			swp_pager_update_freerange(&range, sb->d[i]);
+			if (freed != NULL) {
+				m = (m != NULL && m->pindex == sb->p + i - 1) ?
+				    vm_page_next(m) :
+				    vm_page_lookup(object, sb->p + i);
+				if (m == NULL || vm_page_none_valid(m))
+					fc++;
+			}
+			sb->d[i] = SWAPBLK_NONE;
+		}
+		pindex = sb->p + SWAP_META_PAGES;
+		if (swp_pager_swblk_empty(sb, 0, start) &&
+		    swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {
+			swblk_lookup_remove(object, sb);
+			uma_zfree(swblk_zone, sb);
+		}
+	}
+	swp_pager_freeswapspace(&range);
+out:
+	if (freed != NULL)
+		*freed = fc;
 }
 
 static void
@@ -2296,19 +2364,16 @@ swp_pager_meta_lookup(vm_object_t object, vm_pindex_t pindex)
 	KASSERT((object->flags & OBJ_SWAP) != 0,
 	    ("Lookup object not swappable"));
 
-	sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,
-	    rounddown(pindex, SWAP_META_PAGES));
+	sb = swblk_lookup(object, pindex);
 	if (sb == NULL)
 		return (SWAPBLK_NONE);
 	return (sb->d[pindex % SWAP_META_PAGES]);
 }
 
 /*
- * Returns the least page index which is greater than or equal to the
- * parameter pindex and for which there is a swap block allocated.
- * Returns object's size if the object's type is not swap or if there
- * are no allocated swap blocks for the object after the requested
- * pindex.
+ * Returns the least page index which is greater than or equal to the parameter
+ * pindex and for which there is a swap block allocated.  Returns OBJ_MAX_SIZE
+ * if are no allocated swap blocks for the object after the requested pindex.
  */
 vm_pindex_t
 swap_pager_find_least(vm_object_t object, vm_pindex_t pindex)
@@ -2316,24 +2381,15 @@ swap_pager_find_least(vm_object_t object, vm_pindex_t pindex)
 	struct swblk *sb;
 	int i;
 
-	VM_OBJECT_ASSERT_LOCKED(object);
-	MPASS((object->flags & OBJ_SWAP) != 0);
-
-	if (pctrie_is_empty(&object->un_pager.swp.swp_blks))
-		return (object->size);
-	sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,
-	    rounddown(pindex, SWAP_META_PAGES));
-	if (sb == NULL)
-		return (object->size);
+	if ((sb = swblk_start(object, pindex)) == NULL)
+		return (OBJ_MAX_SIZE);
 	if (sb->p < pindex) {
 		for (i = pindex % SWAP_META_PAGES; i < SWAP_META_PAGES; i++) {
 			if (sb->d[i] != SWAPBLK_NONE)
 				return (sb->p + i);
 		}
-		sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,
-		    roundup(pindex, SWAP_META_PAGES));
-		if (sb == NULL)
-			return (object->size);
+		if ((sb = swblk_next(object, sb)) == NULL)
+			return (OBJ_MAX_SIZE);
 	}
 	for (i = 0; i < SWAP_META_PAGES; i++) {
 		if (sb->d[i] != SWAPBLK_NONE)
@@ -2345,7 +2401,7 @@ swap_pager_find_least(vm_object_t object, vm_pindex_t pindex)
 	 * doesn't map any blocks.
 	 */
 	MPASS(0);
-	return (object->size);
+	return (OBJ_MAX_SIZE);
 }
 
 /*
@@ -2797,11 +2853,8 @@ vmspace_swap_count(struct vmspace *vmspace)
 			goto unlock;
 		pi = OFF_TO_IDX(cur->offset);
 		e = pi + OFF_TO_IDX(cur->end - cur->start);
-		for (;; pi = sb->p + SWAP_META_PAGES) {
-			sb = SWAP_PCTRIE_LOOKUP_GE(
-			    &object->un_pager.swp.swp_blks, pi);
-			if (sb == NULL || sb->p >= e)
-				break;
+		for (sb = swblk_start_limit(object, pi, e);
+		    sb != NULL; sb = swblk_next_limit(object, sb, e)) {
 			for (i = 0; i < SWAP_META_PAGES; i++) {
 				if (sb->p + i < e &&
 				    sb->d[i] != SWAPBLK_NONE)



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202409092146.489LkYBL023066>