Date: Sun, 8 Jun 2025 18:36:48 GMT From: Alan Cox <alc@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 1fee99800a79 - main - vm_page: Retire its listq field Message-ID: <202506081836.558IamHL008409@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by alc: URL: https://cgit.FreeBSD.org/src/commit/?id=1fee99800a79887b9037749a34d09f2acab082c0 commit 1fee99800a79887b9037749a34d09f2acab082c0 Author: Alan Cox <alc@FreeBSD.org> AuthorDate: 2025-05-27 08:27:16 +0000 Commit: Alan Cox <alc@FreeBSD.org> CommitDate: 2025-06-08 18:35:56 +0000 vm_page: Retire its listq field Over the life cycle of a vm_page, its listq field has been used for two distinct purposes. First, linking together all of the pages allocated to a vm_object. Recently, c8d56817b80f ("vm_object: drop memq field") completed the elimination of this use case, using pctrie iterators in place of iteration over the listq. Second, linking together power-of- two-sized chunks of free pages within vm_phys. This change eliminates that use case. In essence, this change reverts vm_phys back to using the plinks.q field, like it did before 5cd29d0f3cda ("Improve VM page queue scalability."), but with a twist to maintain scalability. Just before vm_phys uses the plinks.q field, it ensures that any lazy dequeue from a paging queue, e.g., PQ_ACTIVE, has completed. Typically, the dequeue has completed, so vm_page_dequeue() is infrequently called by vm_freelist_add(). The reason being that vm_phys only needs to use the plinks.q field within the first page of any power-of-two-sized chunk, so the rest of the pages can still have pending dequeues until the chunk is split. This change saves a non-trivial amount of memory, since we have an instance of struct vm_page for every dynamically allocatable physical page. Bump __FreeBSD_version, since third-party modules that use the inline accessors in vm_page.h may need to be recompiled. Reviewed by: dougm, kib, markj Differential Revision: https://reviews.freebsd.org/D50515 --- sys/sys/param.h | 2 +- sys/vm/vm_page.c | 22 +++++++++++++++------- sys/vm/vm_page.h | 1 - sys/vm/vm_phys.c | 20 +++++++++++++++----- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/sys/sys/param.h b/sys/sys/param.h index da2089918323..e167c96cf9f8 100644 --- a/sys/sys/param.h +++ b/sys/sys/param.h @@ -73,7 +73,7 @@ * cannot include sys/param.h and should only be updated here. */ #undef __FreeBSD_version -#define __FreeBSD_version 1500045 +#define __FreeBSD_version 1500046 /* * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD, diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index 79eaf8563208..128a1ef7ca54 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -341,7 +341,7 @@ vm_page_blacklist_add(vm_paddr_t pa, bool verbose) vm_domain_free_unlock(vmd); if (found) { vm_domain_freecnt_inc(vmd, -1); - TAILQ_INSERT_TAIL(&blacklist_head, m, listq); + TAILQ_INSERT_TAIL(&blacklist_head, m, plinks.q); if (verbose) printf("Skipping page with pa 0x%jx\n", (uintmax_t)pa); } @@ -411,7 +411,7 @@ sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS) if (error != 0) return (error); sbuf_new_for_sysctl(&sbuf, NULL, 128, req); - TAILQ_FOREACH(m, &blacklist_head, listq) { + TAILQ_FOREACH(m, &blacklist_head, plinks.q) { sbuf_printf(&sbuf, "%s%#jx", first ? "" : ",", (uintmax_t)m->phys_addr); first = 0; @@ -2470,6 +2470,13 @@ again: } found: + /* + * If the page comes from the free page cache, then it might still + * have a pending deferred dequeue. Specifically, when the page is + * imported from a different pool by vm_phys_alloc_npages(), the + * second, third, etc. pages in a non-zero order set could have + * pending deferred dequeues. + */ vm_page_dequeue(m); vm_page_alloc_check(m); @@ -2536,17 +2543,18 @@ vm_page_alloc_nofree_domain(int domain, int req) return (NULL); } m->ref_count = count - 1; - TAILQ_INSERT_HEAD(&vmd->vmd_nofreeq, m, listq); + TAILQ_INSERT_HEAD(&vmd->vmd_nofreeq, m, plinks.q); VM_CNT_ADD(v_nofree_count, count); } m = TAILQ_FIRST(&vmd->vmd_nofreeq); - TAILQ_REMOVE(&vmd->vmd_nofreeq, m, listq); + TAILQ_REMOVE(&vmd->vmd_nofreeq, m, plinks.q); if (m->ref_count > 0) { vm_page_t m_next; m_next = &m[1]; + vm_page_dequeue(m_next); m_next->ref_count = m->ref_count - 1; - TAILQ_INSERT_HEAD(&vmd->vmd_nofreeq, m_next, listq); + TAILQ_INSERT_HEAD(&vmd->vmd_nofreeq, m_next, plinks.q); m->ref_count = 0; } vm_domain_free_unlock(vmd); @@ -2566,7 +2574,7 @@ vm_page_free_nofree(struct vm_domain *vmd, vm_page_t m) { vm_domain_free_lock(vmd); MPASS(m->ref_count == 0); - TAILQ_INSERT_HEAD(&vmd->vmd_nofreeq, m, listq); + TAILQ_INSERT_HEAD(&vmd->vmd_nofreeq, m, plinks.q); vm_domain_free_unlock(vmd); VM_CNT_ADD(v_nofree_count, 1); } @@ -3971,7 +3979,7 @@ vm_page_dequeue(vm_page_t m) old = vm_page_astate_load(m); do { - if (old.queue == PQ_NONE) { + if (__predict_true(old.queue == PQ_NONE)) { KASSERT((old.flags & PGA_QUEUE_STATE_MASK) == 0, ("%s: page %p has unexpected queue state", __func__, m)); diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index 4bcd8d9f8236..4f82a69ebe25 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -229,7 +229,6 @@ struct vm_page { void *zone; } uma; } plinks; - TAILQ_ENTRY(vm_page) listq; /* pages in same object (O) */ vm_object_t object; /* which object am I in (O) */ vm_pindex_t pindex; /* offset into object (O,P) */ vm_paddr_t phys_addr; /* physical address of page (C) */ diff --git a/sys/vm/vm_phys.c b/sys/vm/vm_phys.c index 95bf6b61fe19..ba16ae551093 100644 --- a/sys/vm/vm_phys.c +++ b/sys/vm/vm_phys.c @@ -393,13 +393,23 @@ static void vm_freelist_add(struct vm_freelist *fl, vm_page_t m, int order, int pool, int tail) { + /* + * The paging queues and the free page lists utilize the same field, + * plinks.q, within the vm_page structure. When a physical page is + * freed, it is lazily removed from the paging queues to reduce the + * cost of removal through batching. Here, we must ensure that any + * deferred dequeue on the physical page has completed before using + * its plinks.q field. + */ + if (__predict_false(vm_page_astate_load(m).queue != PQ_NONE)) + vm_page_dequeue(m); m->order = order; m->pool = pool; if (tail) - TAILQ_INSERT_TAIL(&fl[order].pl, m, listq); + TAILQ_INSERT_TAIL(&fl[order].pl, m, plinks.q); else - TAILQ_INSERT_HEAD(&fl[order].pl, m, listq); + TAILQ_INSERT_HEAD(&fl[order].pl, m, plinks.q); fl[order].lcnt++; } @@ -407,7 +417,7 @@ static void vm_freelist_rem(struct vm_freelist *fl, vm_page_t m, int order) { - TAILQ_REMOVE(&fl[order].pl, m, listq); + TAILQ_REMOVE(&fl[order].pl, m, plinks.q); fl[order].lcnt--; m->order = VM_NFREEORDER; } @@ -1582,7 +1592,7 @@ vm_phys_find_freelist_contig(struct vm_freelist *fl, u_long npages, * check if there are enough free blocks starting at a properly aligned * block. Thus, no block is checked for free-ness more than twice. */ - TAILQ_FOREACH(m, &fl[max_order].pl, listq) { + TAILQ_FOREACH(m, &fl[max_order].pl, plinks.q) { /* * Skip m unless it is first in a sequence of free max page * blocks >= low in its segment. @@ -1655,7 +1665,7 @@ vm_phys_find_queues_contig( for (oind = order; oind < VM_NFREEORDER; oind++) { for (pind = vm_default_freepool; pind < VM_NFREEPOOL; pind++) { fl = (*queues)[pind]; - TAILQ_FOREACH(m_ret, &fl[oind].pl, listq) { + TAILQ_FOREACH(m_ret, &fl[oind].pl, plinks.q) { /* * Determine if the address range starting at pa * is within the given range, satisfies the
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202506081836.558IamHL008409>