Date: Sun, 15 Sep 2024 06:07:33 GMT From: Doug Moore <dougm@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 4ccad5452058 - main - swap_pager: avoid meta_transfer race Message-ID: <202409150607.48F67XXO029316@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by dougm: URL: https://cgit.FreeBSD.org/src/commit/?id=4ccad5452058749a2a6ee5865ff485a3f11b20df commit 4ccad5452058749a2a6ee5865ff485a3f11b20df Author: Doug Moore <dougm@FreeBSD.org> AuthorDate: 2024-09-15 06:00:00 +0000 Commit: Doug Moore <dougm@FreeBSD.org> CommitDate: 2024-09-15 06:00:00 +0000 swap_pager: avoid meta_transfer race Function swp_pager_meta_transfer expects that after dropping and reacquiring an object lock, the swap block it's processing still exists, and has not been removed from the trie and freed. Rewrite to avoid depending on that, by scrubbing, removing and freeing it before releasing the lock. Reviewed by: alc, markj Differential Revision: https://reviews.freebsd.org/D46629 --- sys/vm/swap_pager.c | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c index c7a9f16a2953..64ae4ad78786 100644 --- a/sys/vm/swap_pager.c +++ b/sys/vm/swap_pager.c @@ -2199,9 +2199,11 @@ swp_pager_meta_transfer(vm_object_t srcobject, vm_object_t dstobject, { struct page_range range; struct swblk *sb; - daddr_t blk; + daddr_t blk, d[SWAP_META_PAGES]; vm_pindex_t offset, last; - int i, limit, start; + int d_mask, i, limit, start; + _Static_assert(8 * sizeof(d_mask) >= SWAP_META_PAGES, + "d_mask not big enough"); VM_OBJECT_ASSERT_WLOCKED(srcobject); VM_OBJECT_ASSERT_WLOCKED(dstobject); @@ -2210,39 +2212,50 @@ swp_pager_meta_transfer(vm_object_t srcobject, vm_object_t dstobject, return; swp_pager_init_freerange(&range); + d_mask = 0; offset = pindex; last = pindex + count; sb = swblk_start_limit(srcobject, pindex, last); start = (sb != NULL && sb->p < pindex) ? pindex - sb->p : 0; for (; sb != NULL; sb = swblk_start_limit(srcobject, pindex, last), start = 0) { - limit = MIN(last - sb->p, SWAP_META_PAGES); + pindex = sb->p; + MPASS(d_mask == 0); + limit = MIN(last - pindex, SWAP_META_PAGES); for (i = start; i < limit; i++) { if (sb->d[i] == SWAPBLK_NONE) continue; blk = swp_pager_meta_build(dstobject, - sb->p + i - offset, sb->d[i], true); + pindex + i - offset, sb->d[i], true); if (blk == sb->d[i]) { /* - * Destination has no swapblk and is not - * resident, so transfer source. - * swp_pager_meta_build() failed memory - * allocation already, likely to sleep in retry. + * Failed memory allocation stopped transfer; + * save this block for transfer with lock + * released. */ - VM_OBJECT_WUNLOCK(srcobject); - swp_pager_meta_build(dstobject, - sb->p + i - offset, sb->d[i], false); - VM_OBJECT_WLOCK(srcobject); + d[i] = blk; + d_mask |= 1 << i; } else if (blk != SWAPBLK_NONE) swp_pager_update_freerange(&range, sb->d[i]); sb->d[i] = SWAPBLK_NONE; } - pindex = sb->p + SWAP_META_PAGES; if (swp_pager_swblk_empty(sb, 0, start) && swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) { swblk_lookup_remove(srcobject, sb); uma_zfree(swblk_zone, sb); } + if (d_mask != 0) { + /* Finish block transfer, with the lock released. */ + VM_OBJECT_WUNLOCK(srcobject); + do { + i = ffs(d_mask) - 1; + swp_pager_meta_build(dstobject, + pindex + i - offset, d[i], false); + d_mask &= ~(1 << i); + } while (d_mask != 0); + VM_OBJECT_WLOCK(srcobject); + } + pindex += SWAP_META_PAGES; } swp_pager_freeswapspace(&range); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202409150607.48F67XXO029316>