Date: Mon, 9 Sep 2024 21:46:34 GMT From: Doug Moore <dougm@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 942647057295 - main - swap_pager: clean up pctrie usage Message-ID: <202409092146.489LkYBL023066@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by dougm: URL: https://cgit.FreeBSD.org/src/commit/?id=942647057295cd4720372dd85bcf6d1b52fbf38e commit 942647057295cd4720372dd85bcf6d1b52fbf38e Author: Doug Moore <dougm@FreeBSD.org> AuthorDate: 2024-09-09 21:44:13 +0000 Commit: Doug Moore <dougm@FreeBSD.org> CommitDate: 2024-09-09 21:44:13 +0000 swap_pager: clean up pctrie usage Define wrapper functions for the pctrie operations specific to swap_pager, to hide some verbose details. Separate the meta_transfer and meta_free functions into separate functions. Reviewed by: kib (previous version) Tested by: pho (previous version) Differential Revision: https://reviews.freebsd.org/D46315 --- sys/vm/swap_pager.c | 231 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 142 insertions(+), 89 deletions(-) diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c index 59d947c71279..c7a9f16a2953 100644 --- a/sys/vm/swap_pager.c +++ b/sys/vm/swap_pager.c @@ -491,7 +491,7 @@ static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t, static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t, vm_size_t *); static void swp_pager_meta_transfer(vm_object_t src, vm_object_t dst, - vm_pindex_t pindex, vm_pindex_t count, vm_size_t *freed); + vm_pindex_t pindex, vm_pindex_t count); static void swp_pager_meta_free_all(vm_object_t); static daddr_t swp_pager_meta_lookup(vm_object_t, vm_pindex_t); @@ -531,6 +531,59 @@ swblk_trie_free(struct pctrie *ptree, void *node) PCTRIE_DEFINE(SWAP, swblk, p, swblk_trie_alloc, swblk_trie_free); +static struct swblk * +swblk_lookup(vm_object_t object, vm_pindex_t pindex) +{ + return (SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, + rounddown(pindex, SWAP_META_PAGES))); +} + +static struct swblk * +swblk_start(vm_object_t object, vm_pindex_t pindex) +{ + return (SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, + rounddown(pindex, SWAP_META_PAGES))); +} + +static struct swblk * +swblk_next(vm_object_t object, struct swblk *sb) +{ + return (swblk_start(object, sb->p + SWAP_META_PAGES)); +} + +static struct swblk * +swblk_start_limit(vm_object_t object, vm_pindex_t pindex, vm_pindex_t limit) +{ + struct swblk *sb = swblk_start(object, pindex); + if (sb != NULL && sb->p < limit) + return (sb); + return (NULL); +} + +static struct swblk * +swblk_next_limit(vm_object_t object, struct swblk *sb, vm_pindex_t limit) +{ + return (swblk_start_limit(object, sb->p + SWAP_META_PAGES, limit)); +} + +static void +swblk_lookup_remove(vm_object_t object, struct swblk *sb) +{ + SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p); +} + +static int +swblk_lookup_insert(vm_object_t object, struct swblk *sb) +{ + return (SWAP_PCTRIE_INSERT(&object->un_pager.swp.swp_blks, sb)); +} + +static bool +swblk_is_empty(vm_object_t object) +{ + return (pctrie_is_empty(&object->un_pager.swp.swp_blks)); +} + /* * SWP_SIZECHECK() - update swap_pager_full indication * @@ -1084,8 +1137,7 @@ swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject, /* * Transfer source to destination. */ - swp_pager_meta_transfer(srcobject, dstobject, offset, dstobject->size, - NULL); + swp_pager_meta_transfer(srcobject, dstobject, offset, dstobject->size); /* * Free left over swap blocks in source. @@ -1218,8 +1270,7 @@ swap_pager_unswapped(vm_page_t m) } swap_pager_unswapped_acct(m); - sb = SWAP_PCTRIE_LOOKUP(&m->object->un_pager.swp.swp_blks, - rounddown(m->pindex, SWAP_META_PAGES)); + sb = swblk_lookup(m->object, m->pindex); if (sb == NULL) return; range.start = sb->d[m->pindex % SWAP_META_PAGES]; @@ -1777,18 +1828,17 @@ u_long swap_pager_swapped_pages(vm_object_t object) { struct swblk *sb; - vm_pindex_t pi; u_long res; int i; VM_OBJECT_ASSERT_LOCKED(object); - if (pctrie_is_empty(&object->un_pager.swp.swp_blks)) + if (swblk_is_empty(object)) return (0); - for (res = 0, pi = 0; (sb = SWAP_PCTRIE_LOOKUP_GE( - &object->un_pager.swp.swp_blks, pi)) != NULL; - pi = sb->p + SWAP_META_PAGES) { + res = 0; + for (sb = swblk_start(object, 0); sb != NULL; + sb = swblk_next(object, sb)) { for (i = 0; i < SWAP_META_PAGES; i++) { if (sb->d[i] != SWAPBLK_NONE) res++; @@ -1835,16 +1885,14 @@ swap_pager_swapoff_object(struct swdevt *sp, vm_object_t object) if (i == SWAP_META_PAGES) { pi = sb->p + SWAP_META_PAGES; if (sb_empty) { - SWAP_PCTRIE_REMOVE( - &object->un_pager.swp.swp_blks, sb->p); + swblk_lookup_remove(object, sb); uma_zfree(swblk_zone, sb); } i = 0; } if (i == 0) { - sb = SWAP_PCTRIE_LOOKUP_GE( - &object->un_pager.swp.swp_blks, pi); + sb = swblk_start(object, pi); if (sb == NULL) break; sb_empty = true; @@ -2020,7 +2068,7 @@ swp_pager_free_empty_swblk(vm_object_t object, struct swblk *sb) { if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) { - SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p); + swblk_lookup_remove(object, sb); uma_zfree(swblk_zone, sb); } } @@ -2050,7 +2098,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk, VM_OBJECT_ASSERT_WLOCKED(object); rdpi = rounddown(pindex, SWAP_META_PAGES); - sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi); + sb = swblk_lookup(object, rdpi); if (sb == NULL) { if (swapblk == SWAPBLK_NONE) return (SWAPBLK_NONE); @@ -2079,8 +2127,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk, } else uma_zwait(swblk_zone); VM_OBJECT_WLOCK(object); - sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, - rdpi); + sb = swblk_lookup(object, rdpi); if (sb != NULL) /* * Somebody swapped out a nearby page, @@ -2090,8 +2137,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk, goto allocated; } for (;;) { - error = SWAP_PCTRIE_INSERT( - &object->un_pager.swp.swp_blks, sb); + error = swblk_lookup_insert(object, sb); if (error == 0) { if (atomic_cmpset_int(&swpctrie_zone_exhausted, 1, 0)) @@ -2113,8 +2159,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk, } else uma_zwait(swpctrie_zone); VM_OBJECT_WLOCK(object); - sb1 = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, - rdpi); + sb1 = swblk_lookup(object, rdpi); if (sb1 != NULL) { uma_zfree(swblk_zone, sb); sb = sb1; @@ -2142,53 +2187,42 @@ allocated: } /* - * SWP_PAGER_META_TRANSFER() - free a range of blocks in the srcobject's swap - * metadata, or transfer it into dstobject. + * SWP_PAGER_META_TRANSFER() - transfer a range of blocks in the srcobject's + * swap metadata into dstobject. * * This routine will free swap metadata structures as they are cleaned * out. */ static void swp_pager_meta_transfer(vm_object_t srcobject, vm_object_t dstobject, - vm_pindex_t pindex, vm_pindex_t count, vm_size_t *moved) + vm_pindex_t pindex, vm_pindex_t count) { struct page_range range; struct swblk *sb; daddr_t blk; - vm_page_t m; vm_pindex_t offset, last; - vm_size_t mc; int i, limit, start; VM_OBJECT_ASSERT_WLOCKED(srcobject); - MPASS(moved == NULL || dstobject == NULL); + VM_OBJECT_ASSERT_WLOCKED(dstobject); - mc = 0; - m = NULL; - if (count == 0 || pctrie_is_empty(&srcobject->un_pager.swp.swp_blks)) - goto out; + if (count == 0 || swblk_is_empty(srcobject)) + return; swp_pager_init_freerange(&range); offset = pindex; last = pindex + count; - for (;;) { - sb = SWAP_PCTRIE_LOOKUP_GE(&srcobject->un_pager.swp.swp_blks, - rounddown(pindex, SWAP_META_PAGES)); - if (sb == NULL || sb->p >= last) - break; - start = pindex > sb->p ? pindex - sb->p : 0; - limit = last - sb->p < SWAP_META_PAGES ? last - sb->p : - SWAP_META_PAGES; + sb = swblk_start_limit(srcobject, pindex, last); + start = (sb != NULL && sb->p < pindex) ? pindex - sb->p : 0; + for (; sb != NULL; + sb = swblk_start_limit(srcobject, pindex, last), start = 0) { + limit = MIN(last - sb->p, SWAP_META_PAGES); for (i = start; i < limit; i++) { - blk = sb->d[i]; - if (blk == SWAPBLK_NONE) + if (sb->d[i] == SWAPBLK_NONE) continue; - if (dstobject == NULL || - (blk = swp_pager_meta_build(dstobject, - sb->p + i - offset, blk, true), - blk != sb->d[i] && blk != SWAPBLK_NONE)) - swp_pager_update_freerange(&range, sb->d[i]); - else if (blk == sb->d[i]) { + blk = swp_pager_meta_build(dstobject, + sb->p + i - offset, sb->d[i], true); + if (blk == sb->d[i]) { /* * Destination has no swapblk and is not * resident, so transfer source. @@ -2197,30 +2231,20 @@ swp_pager_meta_transfer(vm_object_t srcobject, vm_object_t dstobject, */ VM_OBJECT_WUNLOCK(srcobject); swp_pager_meta_build(dstobject, - sb->p + i - offset, blk, false); + sb->p + i - offset, sb->d[i], false); VM_OBJECT_WLOCK(srcobject); - } - if (moved != NULL) { - m = (m != NULL && m->pindex == sb->p + i - 1) ? - vm_page_next(m) : - vm_page_lookup(srcobject, sb->p + i); - if (m == NULL || vm_page_none_valid(m)) - mc++; - } + } else if (blk != SWAPBLK_NONE) + swp_pager_update_freerange(&range, sb->d[i]); sb->d[i] = SWAPBLK_NONE; } pindex = sb->p + SWAP_META_PAGES; if (swp_pager_swblk_empty(sb, 0, start) && swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) { - SWAP_PCTRIE_REMOVE(&srcobject->un_pager.swp.swp_blks, - sb->p); + swblk_lookup_remove(srcobject, sb); uma_zfree(swblk_zone, sb); } } swp_pager_freeswapspace(&range); -out: - if (moved != NULL) - *moved = mc; } /* @@ -2237,7 +2261,51 @@ static void swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count, vm_size_t *freed) { - swp_pager_meta_transfer(object, NULL, pindex, count, freed); + struct page_range range; + struct swblk *sb; + vm_page_t m; + vm_pindex_t last; + vm_size_t fc; + int i, limit, start; + + VM_OBJECT_ASSERT_WLOCKED(object); + + fc = 0; + m = NULL; + if (count == 0 || swblk_is_empty(object)) + goto out; + + swp_pager_init_freerange(&range); + last = pindex + count; + sb = swblk_start_limit(object, pindex, last); + start = (sb != NULL && sb->p < pindex) ? pindex - sb->p : 0; + for (; sb != NULL; + sb = swblk_start_limit(object, pindex, last), start = 0) { + limit = MIN(last - sb->p, SWAP_META_PAGES); + for (i = start; i < limit; i++) { + if (sb->d[i] == SWAPBLK_NONE) + continue; + swp_pager_update_freerange(&range, sb->d[i]); + if (freed != NULL) { + m = (m != NULL && m->pindex == sb->p + i - 1) ? + vm_page_next(m) : + vm_page_lookup(object, sb->p + i); + if (m == NULL || vm_page_none_valid(m)) + fc++; + } + sb->d[i] = SWAPBLK_NONE; + } + pindex = sb->p + SWAP_META_PAGES; + if (swp_pager_swblk_empty(sb, 0, start) && + swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) { + swblk_lookup_remove(object, sb); + uma_zfree(swblk_zone, sb); + } + } + swp_pager_freeswapspace(&range); +out: + if (freed != NULL) + *freed = fc; } static void @@ -2296,19 +2364,16 @@ swp_pager_meta_lookup(vm_object_t object, vm_pindex_t pindex) KASSERT((object->flags & OBJ_SWAP) != 0, ("Lookup object not swappable")); - sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, - rounddown(pindex, SWAP_META_PAGES)); + sb = swblk_lookup(object, pindex); if (sb == NULL) return (SWAPBLK_NONE); return (sb->d[pindex % SWAP_META_PAGES]); } /* - * Returns the least page index which is greater than or equal to the - * parameter pindex and for which there is a swap block allocated. - * Returns object's size if the object's type is not swap or if there - * are no allocated swap blocks for the object after the requested - * pindex. + * Returns the least page index which is greater than or equal to the parameter + * pindex and for which there is a swap block allocated. Returns OBJ_MAX_SIZE + * if are no allocated swap blocks for the object after the requested pindex. */ vm_pindex_t swap_pager_find_least(vm_object_t object, vm_pindex_t pindex) @@ -2316,24 +2381,15 @@ swap_pager_find_least(vm_object_t object, vm_pindex_t pindex) struct swblk *sb; int i; - VM_OBJECT_ASSERT_LOCKED(object); - MPASS((object->flags & OBJ_SWAP) != 0); - - if (pctrie_is_empty(&object->un_pager.swp.swp_blks)) - return (object->size); - sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, - rounddown(pindex, SWAP_META_PAGES)); - if (sb == NULL) - return (object->size); + if ((sb = swblk_start(object, pindex)) == NULL) + return (OBJ_MAX_SIZE); if (sb->p < pindex) { for (i = pindex % SWAP_META_PAGES; i < SWAP_META_PAGES; i++) { if (sb->d[i] != SWAPBLK_NONE) return (sb->p + i); } - sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, - roundup(pindex, SWAP_META_PAGES)); - if (sb == NULL) - return (object->size); + if ((sb = swblk_next(object, sb)) == NULL) + return (OBJ_MAX_SIZE); } for (i = 0; i < SWAP_META_PAGES; i++) { if (sb->d[i] != SWAPBLK_NONE) @@ -2345,7 +2401,7 @@ swap_pager_find_least(vm_object_t object, vm_pindex_t pindex) * doesn't map any blocks. */ MPASS(0); - return (object->size); + return (OBJ_MAX_SIZE); } /* @@ -2797,11 +2853,8 @@ vmspace_swap_count(struct vmspace *vmspace) goto unlock; pi = OFF_TO_IDX(cur->offset); e = pi + OFF_TO_IDX(cur->end - cur->start); - for (;; pi = sb->p + SWAP_META_PAGES) { - sb = SWAP_PCTRIE_LOOKUP_GE( - &object->un_pager.swp.swp_blks, pi); - if (sb == NULL || sb->p >= e) - break; + for (sb = swblk_start_limit(object, pi, e); + sb != NULL; sb = swblk_next_limit(object, sb, e)) { for (i = 0; i < SWAP_META_PAGES; i++) { if (sb->p + i < e && sb->d[i] != SWAPBLK_NONE)
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202409092146.489LkYBL023066>