From owner-freebsd-arm@FreeBSD.ORG Sat Aug 18 17:53:04 2012 Return-Path: Delivered-To: arm@FreeBSD.org Received: from mx1.freebsd.org (mx1.freebsd.org [69.147.83.52]) by hub.freebsd.org (Postfix) with ESMTP id 28486106566C for ; Sat, 18 Aug 2012 17:53:04 +0000 (UTC) (envelope-from alc@rice.edu) Received: from mh10.mail.rice.edu (mh10.mail.rice.edu [128.42.201.30]) by mx1.freebsd.org (Postfix) with ESMTP id EC7CE8FC08 for ; Sat, 18 Aug 2012 17:53:00 +0000 (UTC) Received: from mh10.mail.rice.edu (localhost.localdomain [127.0.0.1]) by mh10.mail.rice.edu (Postfix) with ESMTP id 33970604F6 for ; Sat, 18 Aug 2012 12:53:00 -0500 (CDT) Received: from mh10.mail.rice.edu (localhost.localdomain [127.0.0.1]) by mh10.mail.rice.edu (Postfix) with ESMTP id 3205D604ED; Sat, 18 Aug 2012 12:53:00 -0500 (CDT) X-Virus-Scanned: by amavis-2.7.0 at mh10.mail.rice.edu, auth channel Received: from mh10.mail.rice.edu ([127.0.0.1]) by mh10.mail.rice.edu (mh10.mail.rice.edu [127.0.0.1]) (amavis, port 10026) with ESMTP id kOplWS-OGjGz; Sat, 18 Aug 2012 12:53:00 -0500 (CDT) Received: from adsl-216-63-78-18.dsl.hstntx.swbell.net (adsl-216-63-78-18.dsl.hstntx.swbell.net [216.63.78.18]) (using TLSv1 with cipher RC4-MD5 (128/128 bits)) (No client certificate requested) (Authenticated sender: alc) by mh10.mail.rice.edu (Postfix) with ESMTPSA id 9A540603D3; Sat, 18 Aug 2012 12:52:59 -0500 (CDT) Message-ID: <502FD67A.7030003@rice.edu> Date: Sat, 18 Aug 2012 12:52:58 -0500 From: Alan Cox User-Agent: Mozilla/5.0 (X11; FreeBSD i386; rv:8.0) Gecko/20111113 Thunderbird/8.0 MIME-Version: 1.0 To: "arm@freebsd.org" Content-Type: multipart/mixed; boundary="------------020600000709070506010501" Cc: Alan Cox Subject: arm pmap locking X-BeenThere: freebsd-arm@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: Porting FreeBSD to the StrongARM Processor List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 18 Aug 2012 17:53:04 -0000 This is a multi-part message in MIME format. --------------020600000709070506010501 Content-Type: text/plain; charset=ISO-8859-1; format=flowed Content-Transfer-Encoding: 7bit Can someone here please test the attached patch? I'm currently working my way through the various pmap implementations eliminating the use of the page queues lock. Arm is next on my list. Alan --------------020600000709070506010501 Content-Type: text/plain; name="arm_pmap2.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename="arm_pmap2.patch" Index: arm/arm/pmap.c =================================================================== --- arm/arm/pmap.c (revision 239352) +++ arm/arm/pmap.c (working copy) @@ -150,6 +150,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include @@ -408,6 +409,7 @@ static vm_offset_t pmap_kernel_l2ptp_kva; static vm_paddr_t pmap_kernel_l2ptp_phys; static struct vm_object pvzone_obj; static int pv_entry_count=0, pv_entry_max=0, pv_entry_high_water=0; +static struct rwlock pvh_global_lock; /* * This list exists for the benefit of pmap_map_chunk(). It keeps track @@ -866,7 +868,7 @@ pmap_alloc_l2_bucket(pmap_t pm, vm_offset_t va) l1idx = L1_IDX(va); PMAP_ASSERT_LOCKED(pm); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + rw_assert(&pvh_global_lock, RA_WLOCKED); if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { /* * No mapping at this address, as there is @@ -875,19 +877,19 @@ pmap_alloc_l2_bucket(pmap_t pm, vm_offset_t va) */ again_l2table: PMAP_UNLOCK(pm); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); if ((l2 = pmap_alloc_l2_dtable()) == NULL) { - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); return (NULL); } - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); if (pm->pm_l2[L2_IDX(l1idx)] != NULL) { PMAP_UNLOCK(pm); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); uma_zfree(l2table_zone, l2); - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); l2 = pm->pm_l2[L2_IDX(l1idx)]; if (l2 == NULL) @@ -919,16 +921,16 @@ again_l2table: */ again_ptep: PMAP_UNLOCK(pm); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); ptep = (void*)uma_zalloc(l2zone, M_NOWAIT|M_USE_RESERVE); - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); if (l2b->l2b_kva != 0) { /* We lost the race. */ PMAP_UNLOCK(pm); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); uma_zfree(l2zone, ptep); - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); if (l2b->l2b_kva == 0) goto again_ptep; @@ -1314,7 +1316,7 @@ pmap_fix_cache(struct vm_page *pg, pmap_t pm, vm_o int entries = 0, kentries = 0, uentries = 0; struct pv_entry *pv; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + rw_assert(&pvh_global_lock, RA_WLOCKED); /* the cache gets written back/invalidated on context switch. * therefore, if a user page shares an entry in the same page or @@ -1426,7 +1428,7 @@ pmap_clearbit(struct vm_page *pg, u_int maskbits) u_int oflags; int count = 0; - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); if (maskbits & PVF_WRITE) maskbits |= PVF_MOD; @@ -1436,7 +1438,7 @@ pmap_clearbit(struct vm_page *pg, u_int maskbits) pg->md.pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF)); if (TAILQ_EMPTY(&pg->md.pv_list)) { - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); return (0); } @@ -1572,7 +1574,7 @@ pmap_clearbit(struct vm_page *pg, u_int maskbits) if (maskbits & PVF_WRITE) vm_page_aflag_clear(pg, PGA_WRITEABLE); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); return (count); } @@ -1601,7 +1603,7 @@ pmap_enter_pv(struct vm_page *pg, struct pv_entry int km; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + rw_assert(&pvh_global_lock, RA_WLOCKED); if (pg->md.pv_kva) { /* PMAP_ASSERT_LOCKED(pmap_kernel()); */ @@ -1615,10 +1617,10 @@ pmap_enter_pv(struct vm_page *pg, struct pv_entry TAILQ_INSERT_HEAD(&pg->md.pv_list, pve, pv_list); TAILQ_INSERT_HEAD(&pve->pv_pmap->pm_pvlist, pve, pv_plist); PMAP_UNLOCK(pmap_kernel()); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); if ((pve = pmap_get_pv_entry()) == NULL) panic("pmap_kenter_internal: no pv entries"); - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); if (km) PMAP_LOCK(pmap_kernel()); } @@ -1647,7 +1649,7 @@ pmap_find_pv(struct vm_page *pg, pmap_t pm, vm_off { struct pv_entry *pv; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + rw_assert(&pvh_global_lock, RA_WLOCKED); TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) if (pm == pv->pv_pmap && va == pv->pv_va) break; @@ -1691,7 +1693,7 @@ pmap_nuke_pv(struct vm_page *pg, pmap_t pm, struct { struct pv_entry *pv; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_ASSERT_LOCKED(pm); TAILQ_REMOVE(&pg->md.pv_list, pve, pv_list); TAILQ_REMOVE(&pm->pm_pvlist, pve, pv_plist); @@ -1737,7 +1739,7 @@ pmap_remove_pv(struct vm_page *pg, pmap_t pm, vm_o { struct pv_entry *pve; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + rw_assert(&pvh_global_lock, RA_WLOCKED); pve = TAILQ_FIRST(&pg->md.pv_list); while (pve) { @@ -1771,7 +1773,7 @@ pmap_modify_pv(struct vm_page *pg, pmap_t pm, vm_o u_int flags, oflags; PMAP_ASSERT_LOCKED(pm); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + rw_assert(&pvh_global_lock, RA_WLOCKED); if ((npv = pmap_find_pv(pg, pm, va)) == NULL) return (0); @@ -1878,7 +1880,7 @@ pmap_fault_fixup(pmap_t pm, vm_offset_t va, vm_pro int rv = 0; l1idx = L1_IDX(va); - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); /* @@ -2075,7 +2077,7 @@ pmap_fault_fixup(pmap_t pm, vm_offset_t va, vm_pro rv = 1; out: - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pm); return (rv); } @@ -2400,6 +2402,11 @@ pmap_bootstrap(vm_offset_t firstaddr, vm_offset_t CPU_FILL(&kernel_pmap->pm_active); kernel_pmap->pm_domain = PMAP_DOMAIN_KERNEL; TAILQ_INIT(&kernel_pmap->pm_pvlist); + + /* + * Initialize the global pv list lock. + */ + rw_init(&pvh_global_lock, "pmap pv global"); /* * Reserve some special page table entries/VA space for temporary @@ -2672,7 +2679,7 @@ pmap_remove_pages(pmap_t pmap) vm_page_t m; pt_entry_t *pt; - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); cpu_idcache_wbinv_all(); cpu_l2cache_wbinv_all(); @@ -2701,7 +2708,7 @@ pmap_remove_pages(pmap_t pmap) pmap_free_pv_entry(pv); pmap_free_l2_bucket(pmap, l2b, 1); } - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); cpu_tlb_flushID(); cpu_cpwait(); PMAP_UNLOCK(pmap); @@ -2826,13 +2833,13 @@ pmap_kenter_internal(vm_offset_t va, vm_offset_t p * This expects the physical memory to have vm_page_array entry. */ if (pvzone != NULL && (m = vm_phys_paddr_to_vm_page(pa))) { - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); if (!TAILQ_EMPTY(&m->md.pv_list) || m->md.pv_kva) { /* release vm_page lock for pv_entry UMA */ - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); if ((pve = pmap_get_pv_entry()) == NULL) panic("pmap_kenter_internal: no pv entries"); - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap_kernel()); pmap_enter_pv(m, pve, pmap_kernel(), va, PVF_WRITE | PVF_UNMAN); @@ -2841,7 +2848,7 @@ pmap_kenter_internal(vm_offset_t va, vm_offset_t p } else { m->md.pv_kva = va; } - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); } } @@ -2902,13 +2909,13 @@ pmap_kremove(vm_offset_t va) /* note: should never have to remove an allocation * before the pvzone is initialized. */ - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap_kernel()); if (pvzone != NULL && (m = vm_phys_paddr_to_vm_page(pa)) && (pve = pmap_remove_pv(m, pmap_kernel(), va))) pmap_free_pv_entry(pve); PMAP_UNLOCK(pmap_kernel()); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); va = va & ~PAGE_MASK; cpu_dcache_wbinv_range(va, PAGE_SIZE); cpu_l2cache_wbinv_range(va, PAGE_SIZE); @@ -3126,7 +3133,7 @@ pmap_remove_all(vm_page_t m) ("pmap_remove_all: page %p is not managed", m)); if (TAILQ_EMPTY(&m->md.pv_list)) return; - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); pmap_remove_write(m); curpm = vmspace_pmap(curproc->p_vmspace); while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { @@ -3175,7 +3182,7 @@ pmap_remove_all(vm_page_t m) pmap_tlb_flushD(curpm); } vm_page_aflag_clear(m, PGA_WRITEABLE); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); } @@ -3208,7 +3215,7 @@ pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset return; } - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); /* @@ -3275,7 +3282,7 @@ pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset if (PV_BEEN_REFD(flags)) pmap_tlb_flushD(pm); } - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pm); } @@ -3299,10 +3306,10 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t vm_prot_t prot, boolean_t wired) { - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); pmap_enter_locked(pmap, va, m, prot, wired, M_WAITOK); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } @@ -3322,7 +3329,7 @@ pmap_enter_locked(pmap_t pmap, vm_offset_t va, vm_ vm_paddr_t pa; PMAP_ASSERT_LOCKED(pmap); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + rw_assert(&pvh_global_lock, RA_WLOCKED); if (va == vector_page) { pa = systempage.pv_pa; m = NULL; @@ -3352,9 +3359,9 @@ do_l2b_alloc: if (l2b == NULL) { if (flags & M_WAITOK) { PMAP_UNLOCK(pmap); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); VM_WAIT; - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); goto do_l2b_alloc; } @@ -3594,14 +3601,14 @@ pmap_enter_object(pmap_t pmap, vm_offset_t start, psize = atop(end - start); m = m_start; - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { pmap_enter_locked(pmap, start + ptoa(diff), m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE, M_NOWAIT); m = TAILQ_NEXT(m, listq); } - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } @@ -3618,11 +3625,11 @@ void pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) { - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); pmap_enter_locked(pmap, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE, M_NOWAIT); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } @@ -3640,7 +3647,7 @@ pmap_change_wiring(pmap_t pmap, vm_offset_t va, bo pt_entry_t *ptep, pte; vm_page_t pg; - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); l2b = pmap_get_l2_bucket(pmap, va); KASSERT(l2b, ("No l2b bucket in pmap_change_wiring")); @@ -3649,7 +3656,7 @@ pmap_change_wiring(pmap_t pmap, vm_offset_t va, bo pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); if (pg) pmap_modify_pv(pg, pmap, va, PVF_WIRED, wired ? PVF_WIRED : 0); - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } @@ -3895,7 +3902,7 @@ pmap_remove(pmap_t pm, vm_offset_t sva, vm_offset_ * we lock in the pmap => pv_head direction */ - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); total = 0; while (sva < eva) { @@ -3989,7 +3996,7 @@ pmap_remove(pmap_t pm, vm_offset_t sva, vm_offset_ pmap_free_l2_bucket(pm, l2b, mappings); } - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); if (flushall) cpu_tlb_flushID(); PMAP_UNLOCK(pm); @@ -4405,7 +4412,7 @@ pmap_page_exists_quick(pmap_t pmap, vm_page_t m) KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_page_exists_quick: page %p is not managed", m)); rv = FALSE; - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (pv->pv_pmap == pmap) { rv = TRUE; @@ -4415,7 +4422,7 @@ pmap_page_exists_quick(pmap_t pmap, vm_page_t m) if (loops >= 16) break; } - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); return (rv); } @@ -4434,11 +4441,11 @@ pmap_page_wired_mappings(vm_page_t m) count = 0; if ((m->oflags & VPO_UNMANAGED) != 0) return (count); - vm_page_lock_queues(); + rw_wlock(&pvh_global_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) if ((pv->pv_flags & PVF_WIRED) != 0) count++; - vm_page_unlock_queues(); + rw_wunlock(&pvh_global_lock); return (count); } --------------020600000709070506010501--