Date: Tue, 11 Apr 2006 08:45:24 GMT From: Alan Cox <alc@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 94980 for review Message-ID: <200604110845.k3B8jOc9016523@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=94980 Change 94980 by alc@alc_home on 2006/04/11 08:44:23 IF user/alc/superpages Various fixes to promotion and demotion of kernel pmap pages. Some code clean up and optimization. Catch up with the change to pmap_remove_pages()'s parameters. Affected files ... .. //depot/projects/superpages/src/sys/i386/i386/pmap.c#12 integrate Differences ... ==== //depot/projects/superpages/src/sys/i386/i386/pmap.c#12 (text+ko) ==== @@ -259,6 +259,8 @@ static pv_entry_t get_pv_entry(pmap_t locked_pmap); static void pmap_clear_ptes(vm_page_t m, int bit); +static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va); +static void pmap_promote_pde(pmap_t pmap, vm_offset_t va, reservation_t reserv); static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva); static void pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva); static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva); @@ -281,9 +283,6 @@ static void *pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait); #endif -static void mach_promote(pmap_t pmap, vm_offset_t va, reservation_t reserv); -static boolean_t pmap_demote(pmap_t pmap, pd_entry_t *pde, vm_offset_t va); - CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t)); CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t)); @@ -1187,7 +1186,7 @@ * normal 4K page. */ if (ptepa & PG_PS) { - pmap_demote(pmap, &pmap->pm_pdir[ptepindex], va); + pmap_demote_pde(pmap, &pmap->pm_pdir[ptepindex], va); ptepa = pmap->pm_pdir[ptepindex]; } @@ -1777,15 +1776,21 @@ printf("pmap_remove: superpage at %x to destroy.\n", sva); #endif + + /* + * The TLB entry for a PG_G mapping is + * invalidated by pmap_remove_pde(). + */ + if ((ptpaddr & PG_G) == 0) + anyvalid = 1; pmap_remove_pde(pmap, pmap_pde(pmap, sva), sva); - anyvalid = 1; continue; } else { #ifdef INVARIANTS printf("pmap_remove: superpage at %x to demote !!!\n", sva); #endif - if (!pmap_demote(pmap, pmap_pde(pmap, sva), sva)) { + if (!pmap_demote_pde(pmap, pmap_pde(pmap, sva), sva)) { anyvalid = 1; /* XXX */ continue; } @@ -1862,7 +1867,7 @@ pde = pmap_pde(pmap, pv->pv_va); if (*pde & PG_PS) { printf("pmap_remove_all: superpage to demote !!!\n"); - if (!pmap_demote(pmap, pde, pv->pv_va)) { + if (!pmap_demote_pde(pmap, pde, pv->pv_va)) { /* * All mappings within the same 4mpage were * destroyed and pv was freed. @@ -1991,7 +1996,7 @@ anychanged = 1; continue; } else { - if (!pmap_demote(pmap, pmap_pde(pmap, sva), sva)) { + if (!pmap_demote_pde(pmap, pmap_pde(pmap, sva), sva)) { anychanged = 1; /* XXX */ continue; } @@ -2250,7 +2255,7 @@ printf("%s: pmap %p va %x XXX\n", __func__, pmap, va); #endif KASSERT(m->object->flags & OBJ_SUPERPAGES, ("pmap_enter: xxx")); - mach_promote(pmap, va, m->reserv); + pmap_promote_pde(pmap, va, m->reserv); } sched_unpin(); vm_page_unlock_queues(); @@ -2380,7 +2385,7 @@ #endif KASSERT(m->object->flags & OBJ_SUPERPAGES, ("pmap_enter_quick: xxx")); - mach_promote(pmap, va, m->reserv); + pmap_promote_pde(pmap, va, m->reserv); } out: PMAP_UNLOCK(pmap); @@ -2801,28 +2806,13 @@ npv = TAILQ_NEXT(pv, pv_plist); continue; } - if (sva <= trunc_4mpage(pv->pv_va) && - eva >= round_4mpage(pv->pv_va + 1)) { -#ifdef INVARIANTS - printf("pmap_remove_pages: superpage at %x to destroy.\n", - trunc_4mpage(pv->pv_va)); -#endif - pmap_remove_pde(pmap, pde, trunc_4mpage(pv->pv_va)); - npv = TAILQ_FIRST(&pmap->pm_pvlist); - continue; - } #ifdef INVARIANTS - printf("pmap_remove_pages: superpage at %x to demote !!!\n", - pv->pv_va); + printf("pmap_remove_pages: superpage at %x to destroy.\n", + trunc_4mpage(pv->pv_va)); #endif - if (!pmap_demote(pmap, pde, pv->pv_va)) { - /* - * All mappings within the same 2mpage were - * destroyed and pv was freed. - */ - npv = TAILQ_FIRST(&pmap->pm_pvlist); - continue; - } + pmap_remove_pde(pmap, pde, trunc_4mpage(pv->pv_va)); + npv = TAILQ_FIRST(&pmap->pm_pvlist); + continue; } #ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY @@ -2982,7 +2972,7 @@ if (*pde & PG_PS) { printf("pmap_clear_ptes: superpage to demote !!!\n"); if ((*pde & bit) == 0 || - !pmap_demote(pmap, pde, pv->pv_va)) { + !pmap_demote_pde(pmap, pde, pv->pv_va)) { /* * All mappings within the same 2mpage were * destroyed and pv was freed. @@ -3292,7 +3282,7 @@ #define COMPATIBLE_PTE(a,b) ((a & COMPATIBLE_PTE_MASK) == (b & COMPATIBLE_PTE_MASK)) static void -mach_promote(pmap_t pmap, vm_offset_t va, reservation_t reserv) +pmap_promote_pde(pmap_t pmap, vm_offset_t va, reservation_t reserv) { vm_paddr_t pa; pmap_t allpmaps_entry; @@ -3314,8 +3304,9 @@ pa += PAGE_SIZE; page_pa = PHYS_TO_VM_PAGE(*pte & PG_FRAME); - KASSERT(page_pa->reserv,("mach_promote: page has no reservation")); - KASSERT(page_pa->reserv == reserv,("mach_promote: reservation mismatch")); + KASSERT(page_pa->reserv,("pmap_promote_pde: page has no reservation")); + KASSERT(page_pa->reserv == reserv, + ("pmap_promote_pde: reservation mismatch")); if ((*pte & PG_V) == 0 || !COMPATIBLE_PTE(*pte, flags)) return; @@ -3341,15 +3332,10 @@ /* Invalidate old TLB entries */ pmap_invalidate_all(pmap); - /* - * XXX - * - * File system corruption occurs if pte pages belonging to the - * kernel pmap are freed. - */ - if (pmap != kernel_pmap) { + /* This leaks up to nkpt kernel pmap page table pages. XXX */ + if (pmap != kernel_pmap || tofree->wire_count == NPTEPG) { KASSERT(tofree->wire_count == NPTEPG, - ("pmap_promote: pte page wire count error")); + ("pmap_promote_pde: pte page wire count error")); tofree->wire_count = 0; vm_page_free(tofree); atomic_subtract_int(&cnt.v_wire_count, 1); @@ -3361,8 +3347,9 @@ } static boolean_t -pmap_demote(pmap_t pmap, pd_entry_t *pde0, vm_offset_t va) +pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va) { + pmap_t allpmaps_entry; pd_entry_t save_pde_value, new_pte_value ; pt_entry_t *pte_page_va, *new_pte_va; vm_paddr_t pte_page_pa; @@ -3370,21 +3357,21 @@ mtx_assert(&vm_page_queue_mtx, MA_OWNED); KASSERT(curthread->td_pinned > 0, ("curthread not pinned")); - KASSERT((*pde0 & PG_PS) != 0, - ("pmap_demote: not a superpage, impossible to demote")); + KASSERT((*pde & PG_PS) != 0, + ("pmap_demote_pde: not a superpage, impossible to demote")); /* STEP 1 * Allocate the PTE page */ if ((pte_page = vm_page_alloc(NULL, va >> PDRSHIFT, VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL | VM_ALLOC_WIRED)) == NULL) { - pmap_remove_pde(pmap, pde0, trunc_4mpage(va)); + pmap_remove_pde(pmap, pde, trunc_4mpage(va)); pmap_invalidate_all(pmap); return (FALSE); } pte_page->wire_count += NPTEPG - 1; KASSERT(pte_page->wire_count == NPTEPG, - ("pmap_demote: page table page %p has wire count %d", + ("pmap_demote_pde: page table page %p has wire count %d", pte_page, pte_page->wire_count)); if (pmap != kernel_pmap) pmap->pm_stats.resident_count++; @@ -3407,7 +3394,7 @@ #endif PMAP1unchanged++; pte_page_va = PADDR1; - pte_page_pa |= PG_U | PG_RW | PG_V | PG_A | PG_M; + pte_page_pa |= PG_M | PG_A | (*pde & PG_U) | PG_RW | PG_V; repeat: @@ -3415,7 +3402,7 @@ * Save the value of the pde entry * Define the value of the first pte entry */ - save_pde_value = *pde0; + save_pde_value = *pde; /* STEP 3 * Fill the PTE page with the physical address of the base pages @@ -3433,10 +3420,23 @@ * If not, assign the new pde value. * If yes, repeat the pte assignment loop. */ - if (!atomic_cmpset_int(pde0, save_pde_value, pte_page_pa)) - goto repeat; + if (pmap == kernel_pmap) { + mtx_lock_spin(&allpmaps_lock); + LIST_FOREACH(allpmaps_entry, &allpmaps, pm_list) { + pde = pmap_pde(allpmaps_entry, va); + KASSERT(COMPATIBLE_PTE(*pde, save_pde_value), + ("pmap_demote_pde: pde was %#jx, expected %#jx", + (uintmax_t)*pde, (uintmax_t)save_pde_value)); + pde_store(pde, pte_page_pa); + } + mtx_unlock_spin(&allpmaps_lock); + } else { + if (!atomic_cmpset_int(pde, save_pde_value, pte_page_pa)) + goto repeat; + } - pmap_invalidate_all(pmap); + /* Invalidate the recursive mapping of the page table page. */ + pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va)); return (TRUE); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200604110845.k3B8jOc9016523>