Date: Fri, 07 Aug 2026 02:52:39 +0000 From: Justin Hibbits <jhibbits@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: b83204edf101 - main - powerpc/pmap: Support booke64 kernel pmap growing Message-ID: <6a754877.1a3f7.3865dab@gitrepo.freebsd.org>
index | next in thread | raw e-mail
The branch main has been updated by jhibbits: URL: https://cgit.FreeBSD.org/src/commit/?id=b83204edf101d2c8ad40005ca2fe9a76796cd4f1 commit b83204edf101d2c8ad40005ca2fe9a76796cd4f1 Author: Justin Hibbits <jhibbits@FreeBSD.org> AuthorDate: 2026-08-05 03:47:27 +0000 Commit: Justin Hibbits <jhibbits@FreeBSD.org> CommitDate: 2026-08-07 02:52:28 +0000 powerpc/pmap: Support booke64 kernel pmap growing In preparation of increasing the KVA on powerpc64 to 2TB to mirror amd64's, rework the 64-bit Book-E pmap to not allocate all page table pages at boot time, since that would be a waste of a lot of memory. Instead, allocate all page table pages for the higher levels, leaving the leaves (page directories) for dynamic allocation. This cuts the boot-time page table size down from ~64MB to ~8MB with the current 32GB KVA size, and bumping to 2TB KVA the boot-time page table is still ~8MB instead of ballooning to ~4GB of mostly wasted space. --- sys/powerpc/booke/pmap.c | 18 +++++++-- sys/powerpc/booke/pmap_64.c | 95 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 101 insertions(+), 12 deletions(-) diff --git a/sys/powerpc/booke/pmap.c b/sys/powerpc/booke/pmap.c index 545416921d09..ce16566a6f13 100644 --- a/sys/powerpc/booke/pmap.c +++ b/sys/powerpc/booke/pmap.c @@ -359,6 +359,9 @@ static int mmu_booke_decode_kernel_ptr(vm_offset_t addr, static void mmu_booke_page_array_startup(long); static bool mmu_booke_page_is_mapped(vm_page_t m); static bool mmu_booke_ps_enabled(pmap_t pmap); +#ifdef __powerpc64__ +static int mmu_booke_growkernel(vm_offset_t); +#endif static struct pmap_funcs mmu_booke_methods = { /* pmap dispatcher interface */ @@ -402,6 +405,9 @@ static struct pmap_funcs mmu_booke_methods = { .page_array_startup = mmu_booke_page_array_startup, .page_is_mapped = mmu_booke_page_is_mapped, .ps_enabled = mmu_booke_ps_enabled, +#ifdef __powerpc64__ + .growkernel_nopanic = mmu_booke_growkernel, +#endif /* Internal interfaces */ .bootstrap = mmu_booke_bootstrap, @@ -700,7 +706,6 @@ mmu_booke_bootstrap(vm_offset_t start, vm_offset_t kernelend) (uintptr_t)msgbufp, data_end); data_end = round_page(data_end); - data_end = round_page(mmu_booke_alloc_kernel_pgtables(data_end)); /* Retrieve phys/avail mem regions */ mem_regions(&physmem_regions, &physmem_regions_sz, @@ -709,7 +714,6 @@ mmu_booke_bootstrap(vm_offset_t start, vm_offset_t kernelend) if (PHYS_AVAIL_ENTRIES < availmem_regions_sz) panic("mmu_booke_bootstrap: phys_avail too small"); - data_end = round_page(data_end); vm_page_array = (vm_page_t)data_end; /* * Get a rough idea (upper bound) on the size of the page array. The @@ -723,6 +727,14 @@ mmu_booke_bootstrap(vm_offset_t start, vm_offset_t kernelend) sz = (round_page(sz) / (PAGE_SIZE + sizeof(struct vm_page))); data_end += round_page(sz * sizeof(struct vm_page)); + /* + * Reserve kernel page-table pages last, so their reservation size can + * be computed from the final bootstrap data_end (on 64-bit, only leaf + * ptbls covering [VM_MIN_KERNEL_ADDRESS, data_end + slack] are + * pre-allocated; the rest are added on demand by pmap_growkernel()). + */ + data_end = round_page(mmu_booke_alloc_kernel_pgtables(data_end)); + /* Pre-round up to 1MB. This wastes some space, but saves TLB entries */ data_end = roundup2(data_end, 1 << 20); @@ -1216,7 +1228,7 @@ mmu_booke_kremove(vm_offset_t va) pte = pte_find(kernel_pmap, va); - if (!PTE_ISVALID(pte)) { + if (pte == NULL || !PTE_ISVALID(pte)) { CTR1(KTR_PMAP, "%s: invalid pte", __func__); return; diff --git a/sys/powerpc/booke/pmap_64.c b/sys/powerpc/booke/pmap_64.c index 97f9a5967167..3f2db07ce931 100644 --- a/sys/powerpc/booke/pmap_64.c +++ b/sys/powerpc/booke/pmap_64.c @@ -113,6 +113,13 @@ unsigned int kernel_pdirs; static uma_zone_t ptbl_root_zone; static pte_t ****kernel_ptbl_root; +/* + * Slack space beyond the bootstrap data_end for which leaf ptbls are + * pre-allocated. Must cover post-alloc growth (1MB roundup + TLB1 grow + + * kstack + guard pages). Rounded up to PDIR_SIZE by the reservation code. + */ +#define BOOTSTRAP_LEAF_SLACK (2 * PDIR_SIZE) + /* * Base of the pmap_mapdev() region. On 32-bit it immediately follows the * userspace address range. On On 64-bit it's far above, at (1 << 63), and @@ -262,14 +269,16 @@ get_pgtbl_page(pmap_t pmap, void **ptr_tbl, uint32_t index, vm_page_t m; page = ptr_tbl[index]; - KASSERT(page != 0 || pmap != kernel_pmap, - ("NULL page table page found in kernel pmap!")); if (page == NULL) { page = mmu_booke_alloc_page(pmap, index, nosleep); if (ptr_tbl[index] == NULL) { *isnew = true; ptr_tbl[index] = page; - if (hold_parent) { + /* + * Kernel page-table pages are never freed, so we do + * not maintain refcounts on their parents. + */ + if (hold_parent && pmap != kernel_pmap) { m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)ptr_tbl)); m->ref_count++; } @@ -518,7 +527,8 @@ kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr) { pte_t *pte; vm_size_t kva_size; - int kernel_pdirs, kernel_pgtbls, pdir_l1s; + int kernel_pdirs, pdir_l1s; + unsigned int kernel_pgtbls; vm_offset_t va, l1_va, pdir_va, ptbl_va; int i, j, k; @@ -526,7 +536,8 @@ kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr) kernel_pmap->pm_root = kernel_ptbl_root; pdir_l1s = howmany(kva_size, PG_ROOT_SIZE); kernel_pdirs = howmany(kva_size, PDIR_L1_SIZE); - kernel_pgtbls = howmany(kva_size, PDIR_SIZE); + kernel_pgtbls = howmany(kernel_vm_end - VM_MIN_KERNEL_ADDRESS, + PDIR_SIZE); /* Initialize kernel pdir */ l1_va = (vm_offset_t)kernel_ptbl_root + @@ -537,7 +548,8 @@ kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr) printf("ptbl_root_va: %#lx\n", (vm_offset_t)kernel_ptbl_root); printf("l1_va: %#lx (%d entries)\n", l1_va, pdir_l1s); printf("pdir_va: %#lx(%d entries)\n", pdir_va, kernel_pdirs); - printf("ptbl_va: %#lx(%d entries)\n", ptbl_va, kernel_pgtbls); + printf("ptbl_va: %#lx(%u entries)\n", ptbl_va, kernel_pgtbls); + printf("kernel_vm_end: %#lx\n", kernel_vm_end); } va = VM_MIN_KERNEL_ADDRESS; @@ -550,8 +562,18 @@ kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr) kernel_pmap->pm_root[i][j] = (pte_t **)pdir_va; for (k = 0; k < PDIR_NENTRIES && va < VM_MAX_KERNEL_ADDRESS; - k++, va += PDIR_SIZE, ptbl_va += PAGE_SIZE) - kernel_pmap->pm_root[i][j][k] = (pte_t *)ptbl_va; + k++, va += PDIR_SIZE) { + /* + * Only wire up leaf ptbl pages for the + * pre-allocated bootstrap KVA range; the rest + * are populated lazily by mmu_booke_growkernel(). + */ + if (va < kernel_vm_end) { + kernel_pmap->pm_root[i][j][k] = + (pte_t *)ptbl_va; + ptbl_va += PAGE_SIZE; + } + } } } /* @@ -572,16 +594,71 @@ static vm_offset_t mmu_booke_alloc_kernel_pgtables(vm_offset_t data_end) { vm_size_t kva_size = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS; + unsigned int leaves; + kernel_ptbl_root = (pte_t ****)data_end; data_end += round_page(PG_ROOT_NENTRIES * sizeof(pte_t ***)); data_end += howmany(kva_size, PG_ROOT_SIZE) * PAGE_SIZE; data_end += howmany(kva_size, PDIR_L1_SIZE) * PAGE_SIZE; - data_end += howmany(kva_size, PDIR_SIZE) * PAGE_SIZE; + /* + * Reserve leaf page-table pages only for the current bootstrap KVA + * range plus a small slack for post-alloc growth (1MB TLB roundup, + * kstack + guard). Leaves for the rest of KVA are allocated on + * demand by mmu_booke_growkernel(). The two-step howmany() converges + * because leaves themselves add at most one extra leaf per 512 leaves. + */ + leaves = howmany(data_end - VM_MIN_KERNEL_ADDRESS + + BOOTSTRAP_LEAF_SLACK, PDIR_SIZE); + leaves = howmany(data_end - VM_MIN_KERNEL_ADDRESS + + BOOTSTRAP_LEAF_SLACK + leaves * PAGE_SIZE, PDIR_SIZE); + data_end += leaves * PAGE_SIZE; + + kernel_vm_end = VM_MIN_KERNEL_ADDRESS + leaves * PDIR_SIZE; return (data_end); } +/* + * Grow the kernel page-table by allocating additional leaf ptbl pages so + * that pte_find(kernel_pmap, va) can succeed for [kernel_vm_end, addr]. + * + * All upper levels (root, pdir_l1, pdir) are pre-populated for the whole + * KVA at bootstrap, so only the leaf level needs to be allocated here. + */ +static int +mmu_booke_growkernel(vm_offset_t addr) +{ + pte_t **pdir; + vm_page_t m; + vm_offset_t va; + + if (addr <= kernel_vm_end) + return (KERN_SUCCESS); + + addr = roundup2(addr, PDIR_SIZE); + if (addr - 1 >= vm_map_max(kernel_map)) + addr = vm_map_max(kernel_map); + + for (va = kernel_vm_end; va < addr; va += PDIR_SIZE) { + pdir = kernel_pmap->pm_root[PG_ROOT_IDX(va)][PDIR_L1_IDX(va)]; + KASSERT(pdir != NULL, + ("mmu_booke_growkernel: NULL pdir at va %#lx", va)); + if (pdir[PDIR_IDX(va)] != NULL) + continue; + m = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED | + VM_ALLOC_ZERO); + if (m == NULL) { + kernel_vm_end = va; + return (KERN_RESOURCE_SHORTAGE); + } + m->pindex = va >> PDIR_SHIFT; + pdir[PDIR_IDX(va)] = (pte_t *)VM_PAGE_TO_DMAP(m); + } + kernel_vm_end = addr; + return (KERN_SUCCESS); +} + /* * Initialize a preallocated and zeroed pmap structure, * such as one in a vmspace structure.home | help
Want to link to this message? Use this
URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?6a754877.1a3f7.3865dab>