Date: Thu, 4 Nov 2021 01:07:59 GMT From: Konstantin Belousov <kib@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org Subject: git: 2f71c973efdf - stable/13 - amd64 pmap: account for the top-level pages Message-ID: <202111040107.1A417xqk007729@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch stable/13 has been updated by kib: URL: https://cgit.FreeBSD.org/src/commit/?id=2f71c973efdf37be562a3f5ff75adfd2ed8d796d commit 2f71c973efdf37be562a3f5ff75adfd2ed8d796d Author: Konstantin Belousov <kib@FreeBSD.org> AuthorDate: 2021-10-20 01:03:43 +0000 Commit: Konstantin Belousov <kib@FreeBSD.org> CommitDate: 2021-11-04 00:56:39 +0000 amd64 pmap: account for the top-level pages (cherry picked from commit e93b5adb6bb83d487eaa4211ac26e116db748c63) --- sys/amd64/amd64/pmap.c | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index de4cbd1da9e1..c759ff840a24 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -1483,6 +1483,15 @@ pmap_resident_count_adj(pmap_t pmap, int count) pmap->pm_stats.resident_count += count; } +static __inline void +pmap_pt_page_count_pinit(pmap_t pmap, int count) +{ + KASSERT(pmap->pm_stats.resident_count + count >= 0, + ("pmap %p resident count underflow %ld %d", pmap, + pmap->pm_stats.resident_count, count)); + pmap->pm_stats.resident_count += count; +} + static __inline void pmap_pt_page_count_adj(pmap_t pmap, int count) { @@ -4292,13 +4301,24 @@ pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags) vm_paddr_t pmltop_phys; int i; + bzero(&pmap->pm_stats, sizeof pmap->pm_stats); + /* - * Allocate the page directory page. Pass NULL instead of a pointer to - * the pmap here to avoid recording this page in the resident count, as - * optimizations in pmap_remove() depend on this. + * Allocate the page directory page. Pass NULL instead of a + * pointer to the pmap here to avoid calling + * pmap_resident_count_adj() through pmap_pt_page_count_adj(), + * since that requires pmap lock. Instead do the accounting + * manually. + * + * Note that final call to pmap_remove() optimization that + * checks for zero resident_count is basically disabled by + * accounting for top-level page. But the optimization was + * not effective since we started using non-managed mapping of + * the shared page. */ pmltop_pg = pmap_alloc_pt_page(NULL, 0, VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_WAITOK); + pmap_pt_page_count_pinit(pmap, 1); pmltop_phys = VM_PAGE_TO_PHYS(pmltop_pg); pmap->pm_pmltop = (pml5_entry_t *)PHYS_TO_DMAP(pmltop_phys); @@ -4328,11 +4348,13 @@ pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags) pmap_pinit_pml4(pmltop_pg); if ((curproc->p_md.md_flags & P_MD_KPTI) != 0) { /* - * As with pmltop_pg, pass NULL instead of a pointer to - * the pmap to ensure that the PTI page isn't counted. + * As with pmltop_pg, pass NULL instead of a + * pointer to the pmap to ensure that the PTI + * page counted explicitly. */ pmltop_pgu = pmap_alloc_pt_page(NULL, 0, VM_ALLOC_WIRED | VM_ALLOC_WAITOK); + pmap_pt_page_count_pinit(pmap, 1); pmap->pm_pmltopu = (pml4_entry_t *)PHYS_TO_DMAP( VM_PAGE_TO_PHYS(pmltop_pgu)); if (pmap_is_la57(pmap)) @@ -4355,7 +4377,6 @@ pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags) vm_radix_init(&pmap->pm_root); CPU_ZERO(&pmap->pm_active); TAILQ_INIT(&pmap->pm_pvchunk); - bzero(&pmap->pm_stats, sizeof pmap->pm_stats); pmap->pm_flags = flags; pmap->pm_eptgen = 0; @@ -4747,9 +4768,6 @@ pmap_release(pmap_t pmap) vm_page_t m; int i; - KASSERT(pmap->pm_stats.resident_count == 0, - ("pmap_release: pmap %p resident count %ld != 0", - pmap, pmap->pm_stats.resident_count)); KASSERT(vm_radix_is_empty(&pmap->pm_root), ("pmap_release: pmap %p has reserved page table page(s)", pmap)); @@ -4776,15 +4794,21 @@ pmap_release(pmap_t pmap) } pmap_free_pt_page(NULL, m, true); + pmap_pt_page_count_pinit(pmap, -1); if (pmap->pm_pmltopu != NULL) { m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap-> pm_pmltopu)); pmap_free_pt_page(NULL, m, false); + pmap_pt_page_count_pinit(pmap, -1); } if (pmap->pm_type == PT_X86 && (cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0) rangeset_fini(&pmap->pm_pkru); + + KASSERT(pmap->pm_stats.resident_count == 0, + ("pmap_release: pmap %p resident count %ld != 0", + pmap, pmap->pm_stats.resident_count)); } static int
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202111040107.1A417xqk007729>