Date: Wed, 5 Oct 2005 13:48:29 GMT From: Olivier Houchard <cognet@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 84850 for review Message-ID: <200510051348.j95DmTCg040877@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=84850 Change 84850 by cognet@cognet on 2005/10/05 13:47:44 In pmap_remove(), pmap_remove_pages() and pmap_protect(), be smarter when dealing with a superpage than blindly demoting it. Affected files ... .. //depot/projects/superpages/src/sys/arm/arm/pmap.c#4 edit Differences ... ==== //depot/projects/superpages/src/sys/arm/arm/pmap.c#4 (text+ko) ==== @@ -189,7 +189,7 @@ #else /* PMAP_DEBUG */ #define PDEBUG(_lev_,_stat_) /* Nothing */ #define dprintf(x, arg...) -#define PMAP_INLINE __inline +#define PMAP_INLINE __inline #endif /* PMAP_DEBUG */ extern struct pv_addr systempage; @@ -1550,8 +1550,6 @@ pt_entry_t *ptep; vm_paddr_t pa0; - if (!reserv) - return; ptep = &l2b->l2b_kva[l2pte_index(va & L2_L_FRAME)]; pa0 = *ptep & L2_L_FRAME; if ((*ptep & L2_S_FRAME) != pa0) @@ -1596,8 +1594,6 @@ uint16_t l1idx; int i; - if (!reserv) - return; va0 = va & L1_S_ADDR_MASK; l1idx = L1_IDX(va0); pd = &pmap->pm_l1->l1_kva[l1idx]; @@ -1644,8 +1640,12 @@ if (*pt & L2_S_PROT_W) pa |= L1_S_PROT_W; *pd = L1_S_PROTO | pa | pte_l1_s_cache_mode | L1_S_DOM(pmap->pm_domain); +#if 0 bzero(pt, 0x100 * sizeof(*pt)); +#endif pmap_free_l2_bucket(pmap, &l2->l2_bucket[L2_BUCKET(l1idx)], 0x100); + if (pmap != pmap_kernel() && l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva != NULL) + panic("missed"); pmap_tlb_flushID(pmap); } @@ -2043,6 +2043,42 @@ return(pve); /* return removed pve */ } + +static void +pmap_destroy_pv_range(pmap_t pm, vm_paddr_t pa, vm_offset_t sva, + vm_offset_t eva) +{ + struct pv_entry *pve, *pvetmp; + int i = 0; + pve = TAILQ_FIRST(&pm->pm_pvlist); + + while (pve) { + pvetmp = TAILQ_NEXT(pve, pv_plist); + if (pve->pv_va >= sva && pve->pv_va < eva) { + pmap_nuke_pv(PHYS_TO_VM_PAGE(pa + (pve->pv_va - sva)), + pm, pve); + pmap_free_pv_entry(pve); + i++; + } + pve = pvetmp; + } +} + +static int +pmap_clearbit_pv_range(pmap_t pm, vm_offset_t sva, vm_offset_t eva, int bits) +{ + struct pv_entry *pve; + int ret = 0; + + TAILQ_FOREACH(pve, &pm->pm_pvlist, pv_plist) { + if (pve->pv_va >= sva && pve->pv_va < eva) { + ret |= pve->pv_flags; + pve->pv_flags &= ~bits; + } + } + return (ret); +} + /* * * pmap_modify_pv: Update pv flags @@ -2173,6 +2209,8 @@ pt_entry_t *ptep, pte; vm_paddr_t pa; u_int l1idx; + struct pv_entry *pv; + struct vm_page *pg; int rv = 0; #if 0 @@ -2184,7 +2222,11 @@ if (l1pte_section_p(*pl1pd)) { if (user && !(*pl1pd & L1_S_PROT_U)) goto out; - if (ftype == VM_PROT_WRITE && !(*pl1pd & L1_S_PROT_W)) + pg = PHYS_TO_VM_PAGE((*pl1pd & L1_S_FRAME) + + (va & L1_S_OFFSET)); + pv = pmap_find_pv(pg, pm, va); + if (!pv || + (ftype == VM_PROT_WRITE && !(pv->pv_flags & PVF_WRITE))) goto out; if ((*pl1pd & L1_S_DOM_MASK) != L1_S_DOM(pm->pm_domain)) { *pl1pd &= ~L1_S_DOM_MASK; @@ -2236,8 +2278,6 @@ * This looks like a good candidate for "page modified" * emulation... */ - struct pv_entry *pv; - struct vm_page *pg; /* Extract the physical address of the page */ if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) @@ -2802,9 +2842,6 @@ { struct pcb *pcb; - pmap_idcache_wbinv_all(pmap); - pmap_tlb_flushID(pmap); - cpu_cpwait(); LIST_REMOVE(pmap, pm_list); if (vector_page < KERNBASE) { struct pcb *curpcb = PCPU_GET(curpcb); @@ -3043,27 +3080,70 @@ pd_entry_t *pd; vm_paddr_t pa; vm_page_t m; - + int i; + vm_page_lock_queues(); for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) { if (pv->pv_va >= eva || pv->pv_va < sva) { npv = TAILQ_NEXT(pv, pv_plist); continue; } - pd = &pmap->pm_l1->l1_kva[L1_IDX(pv->pv_va)]; - if (l1pte_section_p(*pd)) - pmap_demote(pmap, pv->pv_va); - if (pv->pv_flags & PVF_WIRED) { + if (pv->pv_flags & PVF_WIRED) { /* The page is wired, cannot remove it now. */ npv = TAILQ_NEXT(pv, pv_plist); continue; } + + pd = &pmap->pm_l1->l1_kva[L1_IDX(pv->pv_va)]; + if (l1pte_section_p(*pd)) { + if ((pv->pv_va & L1_S_ADDR_MASK) >= sva && + ((pv->pv_va & L1_S_ADDR_MASK) + L1_S_SIZE) < + eva) { + pmap_destroy_pv_range(pmap, + *pd & L1_S_ADDR_MASK, + pv->pv_va & L1_S_ADDR_MASK, + (pv->pv_va & L1_S_ADDR_MASK) + + L1_S_SIZE); + for (i = 0; i < 0x100; i++) { + m = PHYS_TO_VM_PAGE( + (*pd & L1_S_ADDR_MASK) + + i * PAGE_SIZE); + if (TAILQ_EMPTY(&m->md.pv_list)) + vm_page_flag_clear(m, + PG_WRITEABLE); + } + *pd = 0; + npv = TAILQ_FIRST(&pmap->pm_pvlist); + continue; + } else + pmap_demote(pmap, pv->pv_va); + } pmap->pm_stats.resident_count--; l2b = pmap_get_l2_bucket(pmap, pv->pv_va); KASSERT(l2b != NULL, ("No L2 bucket in pmap_remove_pages")); pt = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; - if ((*pt & L2_TYPE_MASK) == L2_TYPE_L) - pmap_demote(pmap, pv->pv_va); + if ((*pt & L2_TYPE_MASK) == L2_TYPE_L) { + if ((pv->pv_va & L2_L_FRAME) >= sva && + (pv->pv_va & L2_L_FRAME) + L2_L_SIZE < eva) { + pmap_destroy_pv_range(pmap, *pt & L2_L_FRAME, + pv->pv_va & L2_L_FRAME, + (pv->pv_va & L2_L_FRAME) + L2_L_SIZE); + pt = &l2b->l2b_kva[l2pte_index(pv->pv_va & + L2_L_FRAME)]; + for (int i = 0; i < 0x10; i++) { + m = PHYS_TO_VM_PAGE((pt[i] & + L2_L_FRAME) + i * PAGE_SIZE); + if (TAILQ_EMPTY(&m->md.pv_list)) + vm_page_flag_clear(m, + PG_WRITEABLE); + pt[i] = 0; + } + pmap_free_l2_bucket(pmap, l2b, 0x10); + npv = TAILQ_FIRST(&pmap->pm_pvlist); + continue; + } else + pmap_demote(pmap, pv->pv_va); + } pa = *pt & L2_S_FRAME; m = PHYS_TO_VM_PAGE(pa); *pt = 0; @@ -3073,6 +3153,7 @@ if (TAILQ_EMPTY(&m->md.pv_list)) vm_page_flag_clear(m, PG_WRITEABLE); pmap_free_pv_entry(pv); + pmap_free_l2_bucket(pmap, l2b, 1); } vm_page_unlock_queues(); cpu_idcache_wbinv_all(); @@ -3449,6 +3530,8 @@ u_int flags; int flush; int demoted = 0; + int i; + pd_entry_t *pd; if ((prot & VM_PROT_READ) == 0) { mtx_lock(&Giant); @@ -3487,9 +3570,25 @@ if ((sva & L1_S_OFFSET) == 0 && sva + L1_S_SIZE < eva) { /* Change the whole 1MB superpage. */ - pm->pm_l1->l1_kva[L1_IDX(sva)] &= ~ - L1_S_PROT_W; - flush++; + pd = &pm->pm_l1->l1_kva[L1_IDX(sva)]; + if (*pd & L1_S_PROT_W) { + *pd &= ~L1_S_PROT_W; + PTE_SYNC(*pd); + flush++; + flags |= pmap_clearbit_pv_range(pm, sva, + sva + L1_S_SIZE, PVF_WRITE); + for (i = 0; i < 0x100; i++) { + vm_page_t m = + PHYS_TO_VM_PAGE( + (*pd & L1_S_FRAME) + + i * PAGE_SIZE); + pmap_vac_me_harder(m, pm, + sva + i * PAGE_SIZE); + if (pmap_track_modified(sva + + i * PAGE_SIZE)) + vm_page_dirty(m); + } + } sva += L1_S_SIZE; continue; } @@ -3508,8 +3607,36 @@ while (sva < next_bucket) { demoted = 0; if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) { - pmap_demote(pm, sva); - demoted = 1; + if ((sva & L2_L_OFFSET) == 0 && + sva + L2_L_SIZE < eva) { + if (*ptep & L2_L_PROT_W) { + flags |= pmap_clearbit_pv_range( + pm, sva, + sva + L2_L_SIZE, PVF_WRITE); + for (i = 0; i < 0x10; i++) { + vm_page_t m; + + m = PHYS_TO_VM_PAGE( + (ptep[i] & + L2_L_FRAME) + + i * PAGE_SIZE); + ptep[i] &= ~L2_L_PROT_W; + PTE_SYNC(&ptep[i]); + pmap_vac_me_harder(m, + pm, sva + + i * PAGE_SIZE); + if (pmap_track_modified( + sva + i * PAGE_SIZE)) + vm_page_dirty(m); + } + flush++; + } + sva += L2_L_SIZE; + continue; + } else { + pmap_demote(pm, sva); + demoted = 1; + } } if ((pte = *ptep) != 0 && (pte & L2_S_PROT_W) != 0) { struct vm_page *pg; @@ -3600,7 +3727,6 @@ /* There's already a superpage at this address, demote. */ pmap_demote(pmap, va); } else if (l1pte_section_p(l1pd)) { - pmap_free_l2_bucket(pmap, l2b, 0); opg = m; l2b = NULL; } @@ -4150,10 +4276,14 @@ pd_entry_t *pd = &pm->pm_l1->l1_kva[L1_IDX(sva)]; if (l1pte_section_p(*pd)) { /* We can just remove the superpage. */ - if (0 && (sva == (sva & L1_S_ADDR_MASK)) && - (sva + 0x100000 < eva)) { + if ((sva == (sva & L1_S_ADDR_MASK)) && + (sva + L1_S_SIZE < eva)) { + pmap_destroy_pv_range(pm, + *pd & L1_S_FRAME, sva, + sva + L1_S_SIZE); *pd = 0; - sva = sva + 0x100000; + flushall = 1; + sva += L1_S_SIZE; continue; } else { pmap_demote(pm, sva); @@ -4173,10 +4303,23 @@ pt_entry_t pte; vm_paddr_t pa; - if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) + if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) { /* 64KB superpage. */ + if ((sva & L2_L_FRAME) == sva && + sva + L2_L_SIZE < eva) { + pmap_destroy_pv_range(pm, + *ptep & L2_L_FRAME, + sva, sva + L2_L_SIZE); + for (int i = 0; i < 0x10; i++) + ptep[i] = 0; + sva += L2_L_SIZE; + ptep += 0x10; + flushall = 1; + mappings += 0x10; + continue; + } pmap_demote(pm, sva); - + } pte = *ptep; if (pte == 0) { @@ -4206,12 +4349,12 @@ pve = pmap_remove_pv(pg, pm, sva); if (pve) { #if 0 - simple_unlock(&pg->mdpage.pvh_slock); + simple_unlock(&pg->mdpage.pvh_slock); #endif - is_exec = - PV_BEEN_EXECD(pve->pv_flags); - is_refd = - PV_BEEN_REFD(pve->pv_flags); + is_exec = + PV_BEEN_EXECD(pve->pv_flags); + is_refd = + PV_BEEN_REFD(pve->pv_flags); pmap_free_pv_entry(pve); } } @@ -4234,8 +4377,7 @@ } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { /* Nuke everything if needed. */ - pmap_idcache_wbinv_all(pm); - pmap_tlb_flushID(pm); + flushall = 1; /* * Roll back the previous PTE list, @@ -4248,14 +4390,13 @@ *ptep = 0; PTE_SYNC(ptep); cleanlist_idx++; - flushall = 1; } else { *ptep = 0; PTE_SYNC(ptep); - if (is_exec) - pmap_tlb_flushID_SE(pm, sva); - else - if (is_refd) + if (!flushall && is_exec) + pmap_tlb_flushID_SE(pm, sva); + else + if (!flushall && is_refd) pmap_tlb_flushD_SE(pm, sva); } @@ -4272,15 +4413,17 @@ for (cnt = 0; cnt < cleanlist_idx; cnt++) { vm_offset_t clva = cleanlist[cnt].va & ~1; - if (cleanlist[cnt].va & 1) { - pmap_idcache_wbinv_range(pm, - clva, PAGE_SIZE); - pmap_tlb_flushID_SE(pm, clva); - } else { - pmap_dcache_wb_range(pm, - clva, PAGE_SIZE, TRUE, - FALSE); - pmap_tlb_flushD_SE(pm, clva); + if (!flushall) { + if (cleanlist[cnt].va & 1) { + pmap_idcache_wbinv_range(pm, + clva, PAGE_SIZE); + pmap_tlb_flushID_SE(pm, clva); + } else { + pmap_dcache_wb_range(pm, + clva, PAGE_SIZE, TRUE, + FALSE); + pmap_tlb_flushD_SE(pm, clva); + } } *cleanlist[cnt].pte = 0; PTE_SYNC_CURRENT(pm, cleanlist[cnt].pte); @@ -4294,7 +4437,6 @@ * easier to flush the whole cache. */ cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; - pmap_idcache_wbinv_all(pm); flushall = 1; } } @@ -4302,9 +4444,11 @@ pmap_free_l2_bucket(pm, l2b, mappings); } + if (flushall) { + pmap_idcache_wbinv_all(pm); + pmap_tlb_flushID(pm); + } vm_page_unlock_queues(); - if (flushall) - cpu_tlb_flushID(); #if 0 pmap_release_pmap_lock(pm); PMAP_MAP_TO_HEAD_UNLOCK();
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200510051348.j95DmTCg040877>