Date: Tue, 4 Oct 2005 13:13:08 GMT From: Olivier Houchard <cognet@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 84785 for review Message-ID: <200510041313.j94DD8MP001118@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=84785 Change 84785 by cognet@cognet on 2005/10/04 13:13:08 Superpages bits for arm. Affected files ... .. //depot/projects/superpages/src/sys/arm/arm/pmap.c#3 edit .. //depot/projects/superpages/src/sys/arm/include/pmap.h#3 edit Differences ... ==== //depot/projects/superpages/src/sys/arm/arm/pmap.c#3 (text+ko) ==== @@ -1,4 +1,4 @@ -/* From: $NetBSD: pmap.c,v 1.148 2004/04/03 04:35:48 bsh Exp $ */ +//* From: $NetBSD: pmap.c,v 1.148 2004/04/03 04:35:48 bsh Exp $ */ /*- * Copyright 2004 Olivier Houchard. * Copyright 2003 Wasabi Systems, Inc. @@ -168,6 +168,7 @@ #include <vm/vm_map.h> #include <vm/vm_page.h> #include <vm/vm_pageout.h> +#include <vm/vm_reserve.h> #include <vm/vm_extern.h> #include <sys/lock.h> #include <sys/mutex.h> @@ -210,10 +211,16 @@ static int pmap_clearbit(struct vm_page *, u_int); static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vm_offset_t); -static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vm_offset_t); +static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vm_offset_t, int flags); static void pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int); static vm_offset_t kernel_pt_lookup(vm_paddr_t); +static void pmap_promote_small(pmap_t, struct l2_bucket *, + vm_offset_t, reservation_t, int); +static void pmap_promote_large(pmap_t, vm_offset_t va, + reservation_t); +static void pmap_demote(pmap_t, vm_offset_t); + static MALLOC_DEFINE(M_VMPMAP, "pmap", "PMAP L1"); vm_offset_t avail_end; /* PA of last available physical page */ @@ -910,7 +917,7 @@ * the bucket/page in the meantime. */ static struct l2_bucket * -pmap_alloc_l2_bucket(pmap_t pm, vm_offset_t va) +pmap_alloc_l2_bucket(pmap_t pm, vm_offset_t va, int flags) { struct l2_dtable *l2; struct l2_bucket *l2b; @@ -946,7 +953,7 @@ * No L2 page table has been allocated. Chances are, this * is because we just allocated the l2_dtable, above. */ - ptep = (void*)uma_zalloc(l2zone, M_NOWAIT); + ptep = (void*)uma_zalloc(l2zone, flags); l2b->l2b_phys = vtophys(ptep); if (ptep == NULL) { /* @@ -1105,10 +1112,14 @@ */ #ifdef ARM_USE_SMALL_ALLOC pde = &kernel_pmap->pm_l1->l1_kva[L1_IDX(va)]; + if (l1pte_section_p(*pde) && va < 0xd0000000 && va > virtual_avail) + panic("ouin"); if (!l1pte_section_p(*pde)) { #endif l2b = pmap_get_l2_bucket(pmap_kernel(), va); ptep = &l2b->l2b_kva[l2pte_index(va)]; + if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) + panic("fuxor"); pte = *ptep; if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { @@ -1454,8 +1465,18 @@ pv->pv_flags |= PVF_NC; + if (l1pte_section_p( + pm->pm_l1->l1_kva[L1_IDX(pv->pv_va)])) { + pmap_demote(pv->pv_pmap, pv->pv_va); + pv->pv_flags |= PVF_DEMOTED; + } l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); + ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; + if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) { + pmap_demote(pv->pv_pmap, pv->pv_va); + pv->pv_flags |= PVF_DEMOTED; + } pte = *ptep & ~L2_S_CACHE_MASK; if ((va != pv->pv_va || pm != pv->pv_pmap) && @@ -1509,10 +1530,177 @@ *ptep = pte; PTE_SYNC_CURRENT(pv->pv_pmap, ptep); + if (pv->pv_flags & PVF_DEMOTED) { + pmap_promote_small(pv->pv_pmap, l2b, pv->pv_va, + NULL, 0); + pmap_promote_large(pv->pv_pmap, pv->pv_va, + NULL); + pv->pv_flags &= ~PVF_DEMOTED; + } } } } +/* XXX: misleading, promote_small actually promotes to large pages. */ +static void +pmap_promote_small(pmap_t pmap, struct l2_bucket *l2b, vm_offset_t va, + reservation_t reserv, int off) +{ + int i; + pt_entry_t *ptep; + vm_paddr_t pa0; + + if (!reserv) + return; + ptep = &l2b->l2b_kva[l2pte_index(va & L2_L_FRAME)]; + pa0 = *ptep & L2_L_FRAME; + if ((*ptep & L2_S_FRAME) != pa0) + return; + if (reserv != NULL && ((pa0 - off * PAGE_SIZE) + != VM_PAGE_TO_PHYS(reserv->first_page))) + return; + /* First check it is eligible for promotion. */ + for (i = 1; i < 0x10; i++) { + if (ptep[i] != (ptep[i - 1] + PAGE_SIZE)) { + /* There's no way we can do it. */ + return; + } + if ((ptep[i] & pte_l2_s_cache_mask) == + pte_l2_s_cache_mode_pt || + (ptep[i - 1] & pte_l2_s_cache_mask) == + pte_l2_s_cache_mode_pt) + panic("fuck"); + } +#ifdef SP_DEBUG + printf("promoting small %x\n", va); +#endif + if (*ptep & L2_S_PROT_U) + pa0 |= L2_L_PROT_U; + if (*ptep & L2_S_PROT_W) + pa0 |= L2_L_PROT_W; + /* Let's do it. */ + for (i = 0; i < 0x10; i++) + ptep[i] = pa0 | L2_L_PROTO | + pte_l2_l_cache_mode; + pmap_tlb_flushID(pmap); +} + +static void +pmap_promote_large(pmap_t pmap, vm_offset_t va, reservation_t reserv) +{ + pd_entry_t *pd; + pt_entry_t *pt; + vm_offset_t va0; + vm_paddr_t pa; + struct l2_dtable *l2; + uint16_t l1idx; + int i; + + if (!reserv) + return; + va0 = va & L1_S_ADDR_MASK; + l1idx = L1_IDX(va0); + pd = &pmap->pm_l1->l1_kva[l1idx]; + KASSERT(pd != NULL, ("NO pd_entry in pmap_promote_large")); + l2 = pmap->pm_l2[L2_IDX(l1idx)]; + KASSERT(l2 != NULL, ("No l2 bucket in pmap_promote_large")); + pt = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva; + if (l2pte_index(va0) != 0) + panic("rate"); + KASSERT(pt != NULL, ("No pt in pmap_promote_large")); + if ((*pt & L2_TYPE_MASK) == L2_TYPE_L) + pa = *pt & L2_L_FRAME; + else + pa = *pt & L2_S_FRAME; + if (pa != (*pt & L1_S_FRAME)) + return; + if (reserv != NULL && pa != VM_PAGE_TO_PHYS(reserv->first_page)) + return; + for (i = 1; i < 0x100; i++) { + vm_paddr_t pa, pa2; + + if ((pt[i] & L2_TYPE_MASK) == L2_TYPE_L) + pa = (pt[i] & L2_L_FRAME) + (i & 0xf) * PAGE_SIZE; + else + pa = pt[i] & L2_S_FRAME; + if ((pt[i - 1] & pte_l2_s_cache_mask) == pte_l2_s_cache_mode_pt + || (pt[i] & pte_l2_s_cache_mask) == pte_l2_s_cache_mode_pt) + panic("fuck2"); + if ((pt[i - 1] & L2_TYPE_MASK) == L2_TYPE_L) + pa2 = (pt[i - 1] & L2_L_FRAME) + + ((i - 1) & 0xf) * PAGE_SIZE; + else + pa2 = pt[i - 1] & L2_S_FRAME; + if (pa != pa2 + PAGE_SIZE) + /* Nothing much we can do. */ + return; + } +#ifdef SP_DEBUG + printf("promoting large %x\n", va); +#endif + /* Now free the unused pts. */ + if (*pt & L2_S_PROT_U) + pa |= L1_S_PROT_U; + if (*pt & L2_S_PROT_W) + pa |= L1_S_PROT_W; + *pd = L1_S_PROTO | pa | pte_l1_s_cache_mode | L1_S_DOM(pmap->pm_domain); + bzero(pt, 0x100 * sizeof(*pt)); + pmap_free_l2_bucket(pmap, &l2->l2_bucket[L2_BUCKET(l1idx)], 0x100); + pmap_tlb_flushID(pmap); +} + +static void +pmap_demote(pmap_t pmap, vm_offset_t va) +{ + pd_entry_t *pd; + pt_entry_t *pt; + struct l2_bucket *l2b; + vm_offset_t va0; + uint16_t l1idx; + uint16_t demote_size; + vm_paddr_t pa; + int i; + +#ifdef SP_DEBUG + printf("demoting %x\n", va); +#endif + l1idx = L1_IDX(va); + pd = &pmap->pm_l1->l1_kva[l1idx]; + if (l1pte_section_p(*pd)) { + va0 = va & L1_S_ADDR_MASK; + if (pmap == pmap_kernel()) + l2b = pmap_get_l2_bucket(pmap, va0); + else + l2b = pmap_alloc_l2_bucket(pmap, va0, M_NOWAIT); + demote_size = 0x100; + pa = *pd & L1_S_ADDR_MASK; + if (*pd & L1_S_PROT_U) + pa |= L2_S_PROT_U; + if (*pd & L1_S_PROT_W) + pa |= L2_S_PROT_W; + pt = &l2b->l2b_kva[l2pte_index(va0)]; + *pd = l2b->l2b_phys | L1_C_DOM(pmap->pm_domain) | L1_C_PROTO; + l2b->l2b_occupancy += 0x100; + } else { + va0 = va & L2_L_FRAME; + l2b = pmap_get_l2_bucket(pmap, va0); + demote_size = 0x10; + pt = &l2b->l2b_kva[l2pte_index(va0)]; + if ((*pt & L2_TYPE_MASK) != L2_TYPE_L) + return; + pa = *pt & L2_L_FRAME; + if (*pt & L2_L_PROT_U) + pa |= L2_S_PROT_U; + if (*pt & L2_L_PROT_W) + pa |= L2_S_PROT_W; + } + pa |= L2_S_PROTO; + for (i = 0; i < demote_size; i++, pa += PAGE_SIZE) + pt[i] = (pa) | pte_l2_s_cache_mode; + pmap_tlb_flushID(pmap); +} + + /* * Modify pte bits for all ptes corresponding to the given physical address. * We use `maskbits' rather than `clearbits' because we're always passing @@ -1559,9 +1747,17 @@ pmap_acquire_pmap_lock(pm); #endif + + if (l1pte_section_p(pm->pm_l1->l1_kva[L1_IDX(va)])) { + pmap_demote(pm, va); + pv->pv_flags |= PVF_DEMOTED; + } l2b = pmap_get_l2_bucket(pm, va); - ptep = &l2b->l2b_kva[l2pte_index(va)]; + if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) { + pmap_demote(pm, va); + pv->pv_flags |= PVF_DEMOTED; + } npte = opte = *ptep; if (maskbits & (PVF_WRITE|PVF_MOD) && @@ -1675,6 +1871,11 @@ if (PV_BEEN_REFD(oflags)) pmap_tlb_flushD_SE(pm, pv->pv_va); } + if (pv->pv_flags & PVF_DEMOTED) { + pmap_promote_small(pm, l2b, va, NULL, 0); + pmap_promote_large(pm, va, NULL); + pv->pv_flags &= ~PVF_DEMOTED; + } #if 0 pmap_release_pmap_lock(pm); @@ -1713,8 +1914,6 @@ pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, pmap_t pm, vm_offset_t va, u_int flags) { - - pve->pv_pmap = pm; pve->pv_va = va; pve->pv_flags = flags; @@ -1981,6 +2180,19 @@ pmap_acquire_pmap_lock(pm); #endif l1idx = L1_IDX(va); + pl1pd = &pm->pm_l1->l1_kva[l1idx]; + if (l1pte_section_p(*pl1pd)) { + if (user && !(*pl1pd & L1_S_PROT_U)) + goto out; + if (ftype == VM_PROT_WRITE && !(*pl1pd & L1_S_PROT_W)) + goto out; + if ((*pl1pd & L1_S_DOM_MASK) != L1_S_DOM(pm->pm_domain)) { + *pl1pd &= ~L1_S_DOM_MASK; + *pl1pd |= L1_S_DOM(pm->pm_domain); + rv = 1; + goto out; + } + } /* * If there is no l2_dtable for this address, then the process @@ -2011,12 +2223,15 @@ /* * Catch a userland access to the vector page mapped at 0x0 */ - if (user && (pte & L2_S_PROT_U) == 0) + if (user && (((pte & L2_TYPE_MASK) == L2_TYPE_L && + (pte & L2_L_PROT_U) == 0) || (pte & L2_S_PROT_U) == 0)) goto out; - pa = l2pte_pa(pte); + pa = (pte & L2_TYPE_MASK) == L2_TYPE_L ? + ((pte & L2_L_FRAME) | (va & L2_L_OFFSET)) : l2pte_pa(pte); - if ((ftype & VM_PROT_WRITE) && (pte & L2_S_PROT_W) == 0) { + if ((ftype & VM_PROT_WRITE) && (((pte & L2_TYPE_MASK) == L2_TYPE_L && + (pte & L2_L_PROT_W) == 0) || (pte & L2_S_PROT_W) == 0)) { /* * This looks like a good candidate for "page modified" * emulation... @@ -2025,15 +2240,13 @@ struct vm_page *pg; /* Extract the physical address of the page */ - if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) { + if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) goto out; - } /* Get the current flags for this page. */ pv = pmap_find_pv(pg, pm, va); - if (pv == NULL) { + if (pv == NULL) goto out; - } /* * Do the flags say this page is writable? If not then it @@ -2042,9 +2255,8 @@ * PTE. Now we know a write has occurred we can correct this * and also set the modified bit */ - if ((pv->pv_flags & PVF_WRITE) == 0) { + if ((pv->pv_flags & PVF_WRITE) == 0) goto out; - } if (pmap_track_modified(pv->pv_va)) { pg->md.pvh_attrs |= PVF_REF | PVF_MOD; @@ -2059,7 +2271,12 @@ * changing. We've already set the cacheable bits based on * the assumption that we can write to this page. */ - *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W; + if ((pte & L2_TYPE_MASK) == L2_TYPE_L) + *ptep = (pte & ~L2_TYPE_MASK) | L2_L_PROTO | + L2_L_PROT_W; + else + *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO | + L2_S_PROT_W; PTE_SYNC(ptep); rv = 1; } else @@ -2088,7 +2305,6 @@ pg->md.pvh_attrs |= PVF_REF; pv->pv_flags |= PVF_REF; - *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO; PTE_SYNC(ptep); rv = 1; @@ -2212,7 +2428,7 @@ pv_entry_high_water = 9 * (pv_entry_max / 10); l2zone = uma_zcreate("L2 Table", L2_TABLE_SIZE_REAL, pmap_l2ptp_ctor, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); - uma_prealloc(l2zone, 4096); + /*uma_prealloc(l2zone, 4096);*/ l2table_zone = uma_zcreate("L2 Table", sizeof(struct l2_dtable), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); @@ -2823,8 +3039,10 @@ { struct pv_entry *pv, *npv; struct l2_bucket *l2b = NULL; + pt_entry_t *pt; + pd_entry_t *pd; + vm_paddr_t pa; vm_page_t m; - pt_entry_t *pt; vm_page_lock_queues(); for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) { @@ -2832,6 +3050,9 @@ npv = TAILQ_NEXT(pv, pv_plist); continue; } + pd = &pmap->pm_l1->l1_kva[L1_IDX(pv->pv_va)]; + if (l1pte_section_p(*pd)) + pmap_demote(pmap, pv->pv_va); if (pv->pv_flags & PVF_WIRED) { /* The page is wired, cannot remove it now. */ npv = TAILQ_NEXT(pv, pv_plist); @@ -2841,7 +3062,10 @@ l2b = pmap_get_l2_bucket(pmap, pv->pv_va); KASSERT(l2b != NULL, ("No L2 bucket in pmap_remove_pages")); pt = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; - m = PHYS_TO_VM_PAGE(*pt & L2_ADDR_MASK); + if ((*pt & L2_TYPE_MASK) == L2_TYPE_L) + pmap_demote(pmap, pv->pv_va); + pa = *pt & L2_S_FRAME; + m = PHYS_TO_VM_PAGE(pa); *pt = 0; PTE_SYNC(pt); npv = TAILQ_NEXT(pv, pv_plist); @@ -3014,6 +3238,7 @@ TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) pmap_dcache_wb_range(pv->pv_pmap, pv->pv_va, PAGE_SIZE, TRUE, TRUE); } + /* * Add a list of wired pages to the kva * this routine is only used for temporary @@ -3084,6 +3309,8 @@ if (!pmap_get_pde_pte(pmap, addr, &pde, &pte)) return (FALSE); + if (!pte) + return (FALSE); if (*pte == 0) return (TRUE); return (FALSE); @@ -3154,6 +3381,7 @@ pmap_remove_all(vm_page_t m) { pv_entry_t pv; + pd_entry_t *pd; pt_entry_t *ptep, pte; struct l2_bucket *l2b; boolean_t flush = FALSE; @@ -3177,9 +3405,15 @@ if (flush == FALSE && (pv->pv_pmap == curpm || pv->pv_pmap == pmap_kernel())) flush = TRUE; + pd = &pv->pv_pmap->pm_l1->l1_kva[L1_IDX(pv->pv_va)]; + if (l1pte_section_p(*pd)) + pmap_demote(pv->pv_pmap, pv->pv_va); l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); + KASSERT(l2b != NULL, ("No l2 bucket")); ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; + if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) + pmap_demote(pv->pv_pmap, pv->pv_va); pte = *ptep; *ptep = 0; PTE_SYNC_CURRENT(pv->pv_pmap, ptep); @@ -3214,6 +3448,7 @@ vm_offset_t next_bucket; u_int flags; int flush; + int demoted = 0; if ((prot & VM_PROT_READ) == 0) { mtx_lock(&Giant); @@ -3243,10 +3478,25 @@ vm_page_lock_queues(); while (sva < eva) { + demoted = 0; next_bucket = L2_NEXT_BUCKET(sva); if (next_bucket > eva) next_bucket = eva; + if (l1pte_section_p(pm->pm_l1->l1_kva[L1_IDX(sva)])) { + if ((sva & L1_S_OFFSET) == 0 && + sva + L1_S_SIZE < eva) { + /* Change the whole 1MB superpage. */ + pm->pm_l1->l1_kva[L1_IDX(sva)] &= ~ + L1_S_PROT_W; + flush++; + sva += L1_S_SIZE; + continue; + } + pmap_demote(pm, sva); + l2b = pmap_get_l2_bucket(pm, sva); + demoted = 1; + } l2b = pmap_get_l2_bucket(pm, sva); if (l2b == NULL) { sva = next_bucket; @@ -3256,6 +3506,11 @@ ptep = &l2b->l2b_kva[l2pte_index(sva)]; while (sva < next_bucket) { + demoted = 0; + if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) { + pmap_demote(pm, sva); + demoted = 1; + } if ((pte = *ptep) != 0 && (pte & L2_S_PROT_W) != 0) { struct vm_page *pg; u_int f; @@ -3285,6 +3540,10 @@ pmap_tlb_flushD_SE(pm, sva); } + if (demoted) { + pmap_promote_small(pm, l2b, sva, NULL, 0); + pmap_promote_large(pm, sva, NULL); + } sva += PAGE_SIZE; ptep++; } @@ -3322,13 +3581,40 @@ boolean_t wired) { struct l2_bucket *l2b = NULL; - struct vm_page *opg; + struct vm_page *opg = NULL; struct pv_entry *pve = NULL; pt_entry_t *ptep, npte, opte; + pd_entry_t l1pd; u_int nflags; u_int oflags; vm_paddr_t pa; + l1pd = pmap->pm_l1->l1_kva[L1_IDX(va)]; + if (l1pte_section_p(l1pd) && + ((l1pd & L1_S_FRAME) != (VM_PAGE_TO_PHYS(m) & L1_S_FRAME) || + (((VM_PAGE_TO_PHYS(m) & L2_S_FRAME) & L2_S_OFFSET) != + (va & L2_S_OFFSET)) || + ((l1pd & L1_S_PROT_U) != 0) != (pmap != pmap_kernel() && va != + vector_page) || ((l1pd & L1_S_PROT_W) != 0) != + ((prot & VM_PROT_WRITE) != 0))) { + /* There's already a superpage at this address, demote. */ + pmap_demote(pmap, va); + } else if (l1pte_section_p(l1pd)) { + pmap_free_l2_bucket(pmap, l2b, 0); + opg = m; + l2b = NULL; + } + + if (opg == NULL) { + if (pmap == pmap_kernel()) { + l2b = pmap_get_l2_bucket(pmap, va); + if (!l2b) + l2b = pmap_grow_l2_bucket(pmap, va); + } else + l2b = pmap_alloc_l2_bucket(pmap, va, M_NOWAIT); + KASSERT(l2b != NULL, + ("pmap_enter: failed to allocate l2 bucket")); + } vm_page_lock_queues(); if (va == vector_page) { pa = systempage.pv_pa; @@ -3345,31 +3631,40 @@ PDEBUG(1, printf("pmap_enter: pmap = %08x, va = %08x, m = %08x, prot = %x, " "wired = %x\n", (uint32_t) pmap, va, (uint32_t) m, prot, wired)); - if (pmap == pmap_kernel()) { - l2b = pmap_get_l2_bucket(pmap, va); - if (l2b == NULL) - l2b = pmap_grow_l2_bucket(pmap, va); - } else - l2b = pmap_alloc_l2_bucket(pmap, va); - KASSERT(l2b != NULL, - ("pmap_enter: failed to allocate l2 bucket")); - ptep = &l2b->l2b_kva[l2pte_index(va)]; + if (l2b) { + ptep = &l2b->l2b_kva[l2pte_index(va)]; - opte = *ptep; - npte = pa; - oflags = 0; + opte = *ptep; + npte = pa; + oflags = 0; + } else { + npte = oflags = opte = 0; + ptep = NULL; + } if (opte) { + if ((opte & L2_TYPE_MASK) == L2_TYPE_L) { + if (((opte & L2_L_FRAME) | (va & L2_L_OFFSET)) == + VM_PAGE_TO_PHYS(m) && + (((opte & L2_L_PROT_W) != 0) == + (prot & VM_PROT_WRITE) != 0) && + (((opte & L2_L_PROT_U) == 0) == + (pmap == pmap_kernel() || va == vector_page))) + opg = m; + else { + pmap_demote(pmap, va); + opte = *ptep; + } + } /* * There is already a mapping at this address. * If the physical address is different, lookup the * vm_page. */ - if (l2pte_pa(opte) != pa) + if (!opg && l2pte_pa(opte) != pa) opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); - else + else if (!opg) opg = m; - } else - opg = NULL; + } if ((prot & (VM_PROT_ALL)) || (!m || m->md.pvh_attrs & PVF_REF)) { @@ -3492,7 +3787,7 @@ /* * Keep the stats up to date */ - if (opte == 0) { + if (opte == 0 && l2b) { l2b->l2b_occupancy++; pmap->pm_stats.resident_count++; } @@ -3502,7 +3797,7 @@ * If this is just a wiring change, the two PTEs will be * identical, so there's no need to update the page table. */ - if (npte != opte) { + if (l2b && npte != opte && ((opte & L2_TYPE_MASK) != L2_TYPE_L)) { boolean_t is_cached = pmap_is_current(pmap); *ptep = npte; @@ -3539,6 +3834,17 @@ pmap_vac_me_harder(m, pmap, va); + if ((va < VM_MAXUSER_ADDRESS || + m->object == kernel_object || + (m->object == kmem_object && FALSE)) && m->reserv) { + if (m->reserv->refcnt > 0 && + m->reserv->refcnt != SP_LARGE && + (m->reserv->refcnt % SMALL) == 0) + pmap_promote_small(pmap, l2b, va, m->reserv, + m->reserv->refcnt - SMALL); + else if (m->reserv->refcnt == SP_LARGE) + pmap_promote_large(pmap, va, m->reserv); + } } vm_page_unlock_queues(); } @@ -3579,15 +3885,19 @@ void pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired) { +#if 0 struct l2_bucket *l2b; pt_entry_t *ptep, pte; +#endif vm_page_t pg; +#if 0 l2b = pmap_get_l2_bucket(pmap, va); KASSERT(l2b, ("No l2b bucket in pmap_change_wiring")); ptep = &l2b->l2b_kva[l2pte_index(va)]; pte = *ptep; - pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); +#endif + pg = PHYS_TO_VM_PAGE(pmap_extract(pmap, va)); if (pg) pmap_modify_pv(pg, pmap, va, PVF_WIRED, wired); } @@ -3626,10 +3936,6 @@ l1pd = *pl1pd; if (l1pte_section_p(l1pd)) { - /* - * These should only happen for pmap_kernel() - */ - KASSERT(pm == pmap_kernel(), ("huh")); pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); } else { /* @@ -3685,10 +3991,6 @@ vm_page_lock_queues(); if (l1pte_section_p(l1pd)) { - /* - * These should only happen for pmap_kernel() - */ - KASSERT(pmap == pmap_kernel(), ("huh")); pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); if (l1pd & L1_S_PROT_W || (prot & VM_PROT_WRITE) == 0) { m = PHYS_TO_VM_PAGE(pa); @@ -3845,8 +4147,22 @@ l2b = pmap_get_l2_bucket(pm, sva); if (l2b == NULL) { - sva = next_bucket; - continue; + pd_entry_t *pd = &pm->pm_l1->l1_kva[L1_IDX(sva)]; + if (l1pte_section_p(*pd)) { + /* We can just remove the superpage. */ + if (0 && (sva == (sva & L1_S_ADDR_MASK)) && + (sva + 0x100000 < eva)) { + *pd = 0; + sva = sva + 0x100000; + continue; + } else { + pmap_demote(pm, sva); + l2b = pmap_get_l2_bucket(pm, sva); + } + } else { + sva = next_bucket; + continue; + } } ptep = &l2b->l2b_kva[l2pte_index(sva)]; @@ -3857,6 +4173,10 @@ pt_entry_t pte; vm_paddr_t pa; + if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) + /* 64KB superpage. */ + pmap_demote(pm, sva); + pte = *ptep; if (pte == 0) { @@ -4017,7 +4337,6 @@ _arm_bzero((void *)(phys + off), size, IS_PHYSICAL) == 0) return; - /* * Hook in the page, zero it, and purge the cache for that * zeroed page. Invalidate the TLB as needed. @@ -4085,7 +4404,11 @@ l2b = pmap_get_l2_bucket(pmap_kernel(), va); + if (!l2b) + panic("fuck2"); sptep = ptep = &l2b->l2b_kva[l2pte_index(va)]; + if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) + panic("bordel"); while (va < next_bucket) { pte = *ptep; @@ -4826,4 +5149,3 @@ return (NULL); } - ==== //depot/projects/superpages/src/sys/arm/include/pmap.h#3 (text+ko) ==== @@ -479,6 +479,7 @@ #define PVF_EXEC 0x10 /* mapping is executable */ #define PVF_UNC 0x20 /* mapping is 'user' non-cacheable */ #define PVF_KNC 0x40 /* mapping is 'kernel' non-cacheable */ +#define PVF_DEMOTED 0x80 /* Page was demoted. */ #define PVF_NC (PVF_UNC|PVF_KNC) void vector_page_setprot(int);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200510041313.j94DD8MP001118>