Date: Fri, 6 Jul 2012 02:18:49 +0000 (UTC) From: Alan Cox <alc@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r238159 - head/sys/powerpc/aim Message-ID: <201207060218.q662InHT033214@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: alc Date: Fri Jul 6 02:18:49 2012 New Revision: 238159 URL: http://svn.freebsd.org/changeset/base/238159 Log: Replace all uses of the vm page queues lock by a r/w lock that is private to this pmap. Tested by: andreast, jhibbits Modified: head/sys/powerpc/aim/mmu_oea.c Modified: head/sys/powerpc/aim/mmu_oea.c ============================================================================== --- head/sys/powerpc/aim/mmu_oea.c Fri Jul 6 01:32:53 2012 (r238158) +++ head/sys/powerpc/aim/mmu_oea.c Fri Jul 6 02:18:49 2012 (r238159) @@ -125,6 +125,7 @@ __FBSDID("$FreeBSD$"); #include <sys/msgbuf.h> #include <sys/mutex.h> #include <sys/proc.h> +#include <sys/rwlock.h> #include <sys/sched.h> #include <sys/sysctl.h> #include <sys/systm.h> @@ -204,6 +205,17 @@ struct pvo_head *moea_pvo_table; /* pvo struct pvo_head moea_pvo_kunmanaged = LIST_HEAD_INITIALIZER(moea_pvo_kunmanaged); /* list of unmanaged pages */ +/* + * Isolate the global pv list lock from data and other locks to prevent false + * sharing within the cache. + */ +static struct { + struct rwlock lock; + char padding[CACHE_LINE_SIZE - sizeof(struct rwlock)]; +} pvh_global __aligned(CACHE_LINE_SIZE); + +#define pvh_global_lock pvh_global.lock + uma_zone_t moea_upvo_zone; /* zone for pvo entries for unmanaged pages */ uma_zone_t moea_mpvo_zone; /* zone for pvo entries for managed pages */ @@ -455,7 +467,7 @@ static __inline void moea_attr_clear(vm_page_t m, int ptebit) { - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + rw_assert(&pvh_global_lock, RA_WLOCKED); m->md.mdpg_attrs &= ~ptebit; } @@ -470,7 +482,7 @@ static __inline void moea_attr_save(vm_page_t m, int ptebit) { - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + rw_assert(&pvh_global_lock, RA_WLOCKED); m->md.mdpg_attrs |= ptebit; } @@ -859,6 +871,11 @@ moea_bootstrap(mmu_t mmup, vm_offset_t k CPU_FILL(&kernel_pmap->pm_active); RB_INIT(&kernel_pmap->pmap_pvo); + /* + * Initialize the global pv list lock. + */ + rw_init(&pvh_global_lock, "pmap pv global"); + /* * Set up the Open Firmware mappings */ @@ -1066,10 +1083,10 @@ moea_enter(mmu_t mmu, pmap_t pmap, vm_of boolean_t wired) { - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); moea_enter_locked(pmap, va, m, prot, wired); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } @@ -1102,7 +1119,7 @@ moea_enter_locked(pmap_t pmap, vm_offset pvo_flags = PVO_MANAGED; } if (pmap_bootstrapped) - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) != 0 || VM_OBJECT_LOCKED(m->object), @@ -1166,14 +1183,14 @@ moea_enter_object(mmu_t mmu, pmap_t pm, psize = atop(end - start); m = m_start; - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { moea_enter_locked(pm, start + ptoa(diff), m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); m = TAILQ_NEXT(m, listq); } - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pm); } @@ -1182,11 +1199,11 @@ moea_enter_quick(mmu_t mmu, pmap_t pm, v vm_prot_t prot) { - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); moea_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pm); } @@ -1342,7 +1359,7 @@ moea_remove_write(mmu_t mmu, vm_page_t m if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return; - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); lo = moea_attr_fetch(m); powerpc_sync(); LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { @@ -1368,7 +1385,7 @@ moea_remove_write(mmu_t mmu, vm_page_t m vm_page_dirty(m); } vm_page_aflag_clear(m, PGA_WRITEABLE); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); } /* @@ -1409,7 +1426,7 @@ moea_page_set_memattr(mmu_t mmu, vm_page return; } - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); pvo_head = vm_page_to_pvoh(m); lo = moea_calc_wimg(VM_PAGE_TO_PHYS(m), ma); @@ -1429,7 +1446,7 @@ moea_page_set_memattr(mmu_t mmu, vm_page PMAP_UNLOCK(pmap); } m->md.mdpg_cache_attrs = ma; - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); } /* @@ -1543,7 +1560,7 @@ moea_page_exists_quick(mmu_t mmu, pmap_t ("moea_page_exists_quick: page %p is not managed", m)); loops = 0; rv = FALSE; - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { if (pvo->pvo_pmap == pmap) { rv = TRUE; @@ -1552,7 +1569,7 @@ moea_page_exists_quick(mmu_t mmu, pmap_t if (++loops >= 16) break; } - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); return (rv); } @@ -1569,11 +1586,11 @@ moea_page_wired_mappings(mmu_t mmu, vm_p count = 0; if ((m->oflags & VPO_UNMANAGED) != 0) return (count); - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) if ((pvo->pvo_vaddr & PVO_WIRED) != 0) count++; - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); return (count); } @@ -1672,7 +1689,7 @@ moea_protect(mmu_t mmu, pmap_t pm, vm_of return; } - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); key.pvo_vaddr = sva; for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); @@ -1700,7 +1717,7 @@ moea_protect(mmu_t mmu, pmap_t pm, vm_of mtx_unlock(&moea_table_mutex); } } - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pm); } @@ -1766,7 +1783,7 @@ moea_remove(mmu_t mmu, pmap_t pm, vm_off { struct pvo_entry *pvo, *tpvo, key; - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); key.pvo_vaddr = sva; for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); @@ -1775,7 +1792,7 @@ moea_remove(mmu_t mmu, pmap_t pm, vm_off moea_pvo_remove(pvo, -1); } PMAP_UNLOCK(pm); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); } /* @@ -1789,7 +1806,7 @@ moea_remove_all(mmu_t mmu, vm_page_t m) struct pvo_entry *pvo, *next_pvo; pmap_t pmap; - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); pvo_head = vm_page_to_pvoh(m); for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) { next_pvo = LIST_NEXT(pvo, pvo_vlink); @@ -1804,7 +1821,7 @@ moea_remove_all(mmu_t mmu, vm_page_t m) vm_page_dirty(m); } vm_page_aflag_clear(m, PGA_WRITEABLE); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); } /* @@ -2279,7 +2296,7 @@ moea_query_bit(vm_page_t m, int ptebit) if (moea_attr_fetch(m) & ptebit) return (TRUE); - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { /* @@ -2288,7 +2305,7 @@ moea_query_bit(vm_page_t m, int ptebit) */ if (pvo->pvo_pte.pte.pte_lo & ptebit) { moea_attr_save(m, ptebit); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); return (TRUE); } } @@ -2312,13 +2329,13 @@ moea_query_bit(vm_page_t m, int ptebit) mtx_unlock(&moea_table_mutex); if (pvo->pvo_pte.pte.pte_lo & ptebit) { moea_attr_save(m, ptebit); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); return (TRUE); } } } - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); return (FALSE); } @@ -2329,7 +2346,7 @@ moea_clear_bit(vm_page_t m, int ptebit) struct pvo_entry *pvo; struct pte *pt; - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); /* * Clear the cached value. @@ -2363,7 +2380,7 @@ moea_clear_bit(vm_page_t m, int ptebit) pvo->pvo_pte.pte.pte_lo &= ~ptebit; } - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); return (count); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201207060218.q662InHT033214>