Date: Fri, 20 Apr 2012 00:45:24 +0000 (UTC) From: Nathan Whitehorn <nwhitehorn@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-9@freebsd.org Subject: svn commit: r234476 - in stable/9/sys: powerpc/aim vm Message-ID: <201204200045.q3K0jO65014820@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: nwhitehorn Date: Fri Apr 20 00:45:23 2012 New Revision: 234476 URL: http://svn.freebsd.org/changeset/base/234476 Log: MFC r233949,233957,233964,234149,234155: More PMAP concurrency and performance improvements by only flushing icaches when needed and executing ptesync only when architecturally required. Modified: stable/9/sys/powerpc/aim/machdep.c stable/9/sys/powerpc/aim/mmu_oea.c stable/9/sys/powerpc/aim/mmu_oea64.c stable/9/sys/powerpc/aim/moea64_native.c stable/9/sys/vm/vm_page.h Directory Properties: stable/9/sys/ (props changed) Modified: stable/9/sys/powerpc/aim/machdep.c ============================================================================== --- stable/9/sys/powerpc/aim/machdep.c Fri Apr 20 00:34:15 2012 (r234475) +++ stable/9/sys/powerpc/aim/machdep.c Fri Apr 20 00:45:23 2012 (r234476) @@ -400,6 +400,9 @@ powerpc_init(vm_offset_t startkernel, vm cacheline_size = 32; } + /* Make sure the kernel icache is valid before we go too much further */ + __syncicache((caddr_t)startkernel, endkernel - startkernel); + #ifndef __powerpc64__ /* * Figure out whether we need to use the 64 bit PMAP. This works by Modified: stable/9/sys/powerpc/aim/mmu_oea.c ============================================================================== --- stable/9/sys/powerpc/aim/mmu_oea.c Fri Apr 20 00:34:15 2012 (r234475) +++ stable/9/sys/powerpc/aim/mmu_oea.c Fri Apr 20 00:45:23 2012 (r234476) @@ -1055,7 +1055,7 @@ moea_enter_locked(pmap_t pmap, vm_offset struct pvo_head *pvo_head; uma_zone_t zone; vm_page_t pg; - u_int pte_lo, pvo_flags, was_exec; + u_int pte_lo, pvo_flags; int error; if (!moea_initialized) { @@ -1063,13 +1063,11 @@ moea_enter_locked(pmap_t pmap, vm_offset zone = moea_upvo_zone; pvo_flags = 0; pg = NULL; - was_exec = PTE_EXEC; } else { pvo_head = vm_page_to_pvoh(m); pg = m; zone = moea_mpvo_zone; pvo_flags = PVO_MANAGED; - was_exec = 0; } if (pmap_bootstrapped) mtx_assert(&vm_page_queue_mtx, MA_OWNED); @@ -1085,18 +1083,6 @@ moea_enter_locked(pmap_t pmap, vm_offset zone = moea_upvo_zone; } - /* - * If this is a managed page, and it's the first reference to the page, - * clear the execness of the page. Otherwise fetch the execness. - */ - if ((pg != NULL) && ((m->oflags & VPO_UNMANAGED) == 0)) { - if (LIST_EMPTY(pvo_head)) { - moea_attr_clear(pg, PTE_EXEC); - } else { - was_exec = moea_attr_fetch(pg) & PTE_EXEC; - } - } - pte_lo = moea_calc_wimg(VM_PAGE_TO_PHYS(m), pmap_page_get_memattr(m)); if (prot & VM_PROT_WRITE) { @@ -1117,22 +1103,14 @@ moea_enter_locked(pmap_t pmap, vm_offset pte_lo, pvo_flags); /* - * Flush the real page from the instruction cache if this page is - * mapped executable and cacheable and was not previously mapped (or - * was not mapped executable). + * Flush the real page from the instruction cache. This has be done + * for all user mappings to prevent information leakage via the + * instruction cache. moea_pvo_enter() returns ENOENT for the first + * mapping for a page. */ - if (error == 0 && (pvo_flags & PVO_EXECUTABLE) && - (pte_lo & PTE_I) == 0 && was_exec == 0) { - /* - * Flush the real memory from the cache. - */ + if (pmap != kernel_pmap && error == ENOENT && + (pte_lo & (PTE_I | PTE_G)) == 0) moea_syncicache(VM_PAGE_TO_PHYS(m), PAGE_SIZE); - if (pg != NULL) - moea_attr_save(pg, PTE_EXEC); - } - - /* XXX syncicache always until problems are sorted */ - moea_syncicache(VM_PAGE_TO_PHYS(m), PAGE_SIZE); } /* @@ -1454,12 +1432,6 @@ moea_kenter_attr(mmu_t mmu, vm_offset_t panic("moea_kenter: failed to enter va %#x pa %#x: %d", va, pa, error); - /* - * Flush the real memory from the instruction cache. - */ - if ((pte_lo & (PTE_I | PTE_G)) == 0) { - moea_syncicache(pa, PAGE_SIZE); - } PMAP_UNLOCK(kernel_pmap); } Modified: stable/9/sys/powerpc/aim/mmu_oea64.c ============================================================================== --- stable/9/sys/powerpc/aim/mmu_oea64.c Fri Apr 20 00:34:15 2012 (r234475) +++ stable/9/sys/powerpc/aim/mmu_oea64.c Fri Apr 20 00:45:23 2012 (r234476) @@ -283,8 +283,6 @@ static struct pvo_entry *moea64_pvo_find /* * Utility routines. */ -static void moea64_enter_locked(mmu_t, pmap_t, vm_offset_t, - vm_page_t, vm_prot_t, boolean_t); static boolean_t moea64_query_bit(mmu_t, vm_page_t, u_int64_t); static u_int moea64_clear_bit(mmu_t, vm_page_t, u_int64_t); static void moea64_kremove(mmu_t, vm_offset_t); @@ -1167,30 +1165,11 @@ moea64_zero_page_idle(mmu_t mmu, vm_page * target pmap with the protection requested. If specified the page * will be wired down. */ + void moea64_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, boolean_t wired) { - - LOCK_TABLE_WR(); - PMAP_LOCK(pmap); - moea64_enter_locked(mmu, pmap, va, m, prot, wired); - UNLOCK_TABLE_WR(); - PMAP_UNLOCK(pmap); -} - -/* - * Map the given physical page at the specified virtual address in the - * target pmap with the protection requested. If specified the page - * will be wired down. - * - * The table (write) and pmap must be locked. - */ - -static void -moea64_enter_locked(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, - vm_prot_t prot, boolean_t wired) -{ struct pvo_head *pvo_head; uma_zone_t zone; vm_page_t pg; @@ -1210,10 +1189,9 @@ moea64_enter_locked(mmu_t mmu, pmap_t pm pvo_flags = PVO_MANAGED; } - PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) != 0 || VM_OBJECT_LOCKED(m->object), - ("moea64_enter_locked: page %p is not busy", m)); + ("moea64_enter: page %p is not busy", m)); /* XXX change the pvo head for fake pages */ if ((m->oflags & VPO_UNMANAGED) != 0) { @@ -1238,15 +1216,22 @@ moea64_enter_locked(mmu_t mmu, pmap_t pm if (wired) pvo_flags |= PVO_WIRED; + LOCK_TABLE_WR(); + PMAP_LOCK(pmap); error = moea64_pvo_enter(mmu, pmap, zone, pvo_head, va, VM_PAGE_TO_PHYS(m), pte_lo, pvo_flags); + PMAP_UNLOCK(pmap); + UNLOCK_TABLE_WR(); /* * Flush the page from the instruction cache if this page is * mapped executable and cacheable. */ - if ((pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) + if (pmap != kernel_pmap && !(m->aflags & PGA_EXECUTABLE) && + (pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) { + vm_page_aflag_set(m, PGA_EXECUTABLE); moea64_syncicache(mmu, pmap, va, VM_PAGE_TO_PHYS(m), PAGE_SIZE); + } } static void @@ -1307,15 +1292,11 @@ moea64_enter_object(mmu_t mmu, pmap_t pm psize = atop(end - start); m = m_start; - LOCK_TABLE_WR(); - PMAP_LOCK(pm); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { - moea64_enter_locked(mmu, pm, start + ptoa(diff), m, prot & + moea64_enter(mmu, pm, start + ptoa(diff), m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); m = TAILQ_NEXT(m, listq); } - UNLOCK_TABLE_WR(); - PMAP_UNLOCK(pm); } void @@ -1323,12 +1304,8 @@ moea64_enter_quick(mmu_t mmu, pmap_t pm, vm_prot_t prot) { - LOCK_TABLE_WR(); - PMAP_LOCK(pm); - moea64_enter_locked(mmu, pm, va, m, + moea64_enter(mmu, pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); - UNLOCK_TABLE_WR(); - PMAP_UNLOCK(pm); } vm_paddr_t @@ -1664,12 +1641,6 @@ moea64_kenter_attr(mmu_t mmu, vm_offset_ if (error != 0 && error != ENOENT) panic("moea64_kenter: failed to enter va %#zx pa %#zx: %d", va, pa, error); - - /* - * Flush the memory from the instruction cache. - */ - if ((pte_lo & (LPTE_I | LPTE_G)) == 0) - __syncicache((void *)va, PAGE_SIZE); } void @@ -1900,6 +1871,7 @@ static void moea64_pvo_protect(mmu_t mmu, pmap_t pm, struct pvo_entry *pvo, vm_prot_t prot) { uintptr_t pt; + struct vm_page *pg; uint64_t oldlo; PMAP_LOCK_ASSERT(pm, MA_OWNED); @@ -1923,18 +1895,20 @@ moea64_pvo_protect(mmu_t mmu, pmap_t pm else pvo->pvo_pte.lpte.pte_lo |= LPTE_BR; + pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN); + /* * If the PVO is in the page table, update that pte as well. */ - if (pt != -1) { + if (pt != -1) MOEA64_PTE_CHANGE(mmu, pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn); - if ((pvo->pvo_pte.lpte.pte_lo & - (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) { - moea64_syncicache(mmu, pm, PVO_VADDR(pvo), - pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN, - PAGE_SIZE); - } + if (pm != kernel_pmap && pg != NULL && !(pg->aflags & PGA_EXECUTABLE) && + (pvo->pvo_pte.lpte.pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) { + if ((pg->oflags & VPO_UNMANAGED) == 0) + vm_page_aflag_set(pg, PGA_EXECUTABLE); + moea64_syncicache(mmu, pm, PVO_VADDR(pvo), + pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN, PAGE_SIZE); } /* @@ -1943,9 +1917,6 @@ moea64_pvo_protect(mmu_t mmu, pmap_t pm */ if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED && (oldlo & LPTE_PP) != LPTE_BR && !(prot && VM_PROT_WRITE)) { - struct vm_page *pg; - - pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN); if (pg != NULL) { if (pvo->pvo_pte.lpte.pte_lo & LPTE_CHG) vm_page_dirty(pg); @@ -2128,15 +2099,11 @@ moea64_remove(mmu_t mmu, pmap_t pm, vm_o void moea64_remove_all(mmu_t mmu, vm_page_t m) { - struct pvo_head *pvo_head; struct pvo_entry *pvo, *next_pvo; pmap_t pmap; - pvo_head = vm_page_to_pvoh(m); LOCK_TABLE_WR(); - for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) { - next_pvo = LIST_NEXT(pvo, pvo_vlink); - + LIST_FOREACH_SAFE(pvo, vm_page_to_pvoh(m), pvo_vlink, next_pvo) { pmap = pvo->pvo_pmap; PMAP_LOCK(pmap); moea64_pvo_remove(mmu, pvo); @@ -2146,6 +2113,7 @@ moea64_remove_all(mmu_t mmu, vm_page_t m if ((m->aflags & PGA_WRITEABLE) && moea64_is_modified(mmu, m)) vm_page_dirty(m); vm_page_aflag_clear(m, PGA_WRITEABLE); + vm_page_aflag_clear(m, PGA_EXECUTABLE); } /* @@ -2350,6 +2318,7 @@ moea64_pvo_enter(mmu_t mmu, pmap_t pm, u static void moea64_pvo_remove(mmu_t mmu, struct pvo_entry *pvo) { + struct vm_page *pg; uintptr_t pt; PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); @@ -2389,12 +2358,10 @@ moea64_pvo_remove(mmu_t mmu, struct pvo_ /* * Update vm about the REF/CHG bits if the page is managed. */ - if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED && - (pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) { - struct vm_page *pg; + pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN); - pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN); - if (pg != NULL) { + if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED && pg != NULL) { + if ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) { if (pvo->pvo_pte.lpte.pte_lo & LPTE_CHG) vm_page_dirty(pg); if (pvo->pvo_pte.lpte.pte_lo & LPTE_REF) @@ -2402,6 +2369,8 @@ moea64_pvo_remove(mmu_t mmu, struct pvo_ if (LIST_EMPTY(vm_page_to_pvoh(pg))) vm_page_aflag_clear(pg, PGA_WRITEABLE); } + if (LIST_EMPTY(vm_page_to_pvoh(pg))) + vm_page_aflag_clear(pg, PGA_EXECUTABLE); } moea64_pvo_entries--; Modified: stable/9/sys/powerpc/aim/moea64_native.c ============================================================================== --- stable/9/sys/powerpc/aim/moea64_native.c Fri Apr 20 00:34:15 2012 (r234475) +++ stable/9/sys/powerpc/aim/moea64_native.c Fri Apr 20 00:45:23 2012 (r234476) @@ -153,13 +153,10 @@ TLBIE(uint64_t vpn) { vpn &= ~(0xffffULL << 48); #ifdef __powerpc64__ - sched_pin(); - __asm __volatile("ptesync"); mtx_lock(&tlbie_mutex); __asm __volatile("tlbie %0" :: "r"(vpn) : "memory"); mtx_unlock(&tlbie_mutex); __asm __volatile("eieio; tlbsync; ptesync"); - sched_unpin(); #else vpn_hi = (uint32_t)(vpn >> 32); vpn_lo = (uint32_t)vpn; @@ -171,7 +168,6 @@ TLBIE(uint64_t vpn) { mr %1, %0; \ insrdi %1,%5,1,0; \ mtmsrd %1; isync; \ - ptesync; \ \ sld %1,%2,%4; \ or %1,%1,%3; \ @@ -265,7 +261,9 @@ moea64_pte_clear_native(mmu_t mmu, uintp * As shown in Section 7.6.3.2.3 */ pt->pte_lo &= ~ptebit; + sched_pin(); TLBIE(vpn); + sched_unpin(); } static void @@ -295,21 +293,16 @@ moea64_pte_unset_native(mmu_t mmu, uintp { struct lpte *pt = (struct lpte *)pt_cookie; - pvo_pt->pte_hi &= ~LPTE_VALID; - - /* Finish all pending operations */ - isync(); - - /* - * Force the reg & chg bits back into the PTEs. - */ - SYNC(); - /* * Invalidate the pte. */ + isync(); + sched_pin(); + pvo_pt->pte_hi &= ~LPTE_VALID; pt->pte_hi &= ~LPTE_VALID; + PTESYNC(); TLBIE(vpn); + sched_unpin(); /* * Save the reg & chg bits. Modified: stable/9/sys/vm/vm_page.h ============================================================================== --- stable/9/sys/vm/vm_page.h Fri Apr 20 00:34:15 2012 (r234475) +++ stable/9/sys/vm/vm_page.h Fri Apr 20 00:45:23 2012 (r234476) @@ -248,9 +248,13 @@ extern struct vpglocks pa_lock[]; * * PGA_WRITEABLE is set exclusively on managed pages by pmap_enter(). When it * does so, the page must be VPO_BUSY. + * + * PGA_EXECUTABLE may be set by pmap routines, and indicates that a page has + * at least one executable mapping. It is not consumed by the VM layer. */ #define PGA_WRITEABLE 0x01 /* page may be mapped writeable */ #define PGA_REFERENCED 0x02 /* page has been referenced */ +#define PGA_EXECUTABLE 0x04 /* page may be mapped executable */ /* * Page flags. If changed at any other time than page allocation or
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201204200045.q3K0jO65014820>