From owner-svn-src-head@FreeBSD.ORG Sun Apr 25 23:18:02 2010 Return-Path: Delivered-To: svn-src-head@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 61BD6106566B; Sun, 25 Apr 2010 23:18:02 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 4FC9B8FC0C; Sun, 25 Apr 2010 23:18:02 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id o3PNI2gw000532; Sun, 25 Apr 2010 23:18:02 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id o3PNI2E2000530; Sun, 25 Apr 2010 23:18:02 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <201004252318.o3PNI2E2000530@svn.freebsd.org> From: Kip Macy Date: Sun, 25 Apr 2010 23:18:02 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r207210 - head/sys/amd64/amd64 X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 25 Apr 2010 23:18:02 -0000 Author: kmacy Date: Sun Apr 25 23:18:02 2010 New Revision: 207210 URL: http://svn.freebsd.org/changeset/base/207210 Log: Incremental reduction of delta with head_page_lock_2 branch - replace modification of pmap resident_count with pmap_resident_count_{inc,dec} - the pv list is protected by the pmap lock, but in several cases we are relying on the vm page queue mutex, move pv_va read under the pmap lock Modified: head/sys/amd64/amd64/pmap.c Modified: head/sys/amd64/amd64/pmap.c ============================================================================== --- head/sys/amd64/amd64/pmap.c Sun Apr 25 23:08:37 2010 (r207209) +++ head/sys/amd64/amd64/pmap.c Sun Apr 25 23:18:02 2010 (r207210) @@ -404,6 +404,21 @@ pmap_pte(pmap_t pmap, vm_offset_t va) return (pmap_pde_to_pte(pde, va)); } +static __inline void +pmap_resident_count_inc(pmap_t pmap, int count) +{ + + PMAP_LOCK_ASSERT(pmap, MA_OWNED); + pmap->pm_stats.resident_count += count; +} + +static __inline void +pmap_resident_count_dec(pmap_t pmap, int count) +{ + + PMAP_LOCK_ASSERT(pmap, MA_OWNED); + pmap->pm_stats.resident_count -= count; +} PMAP_INLINE pt_entry_t * vtopte(vm_offset_t va) @@ -1477,6 +1492,7 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_of vm_page_t *free) { + PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * unmap the page table page */ @@ -1496,7 +1512,7 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_of pd = pmap_pde(pmap, va); *pd = 0; } - --pmap->pm_stats.resident_count; + pmap_resident_count_dec(pmap, 1); if (m->pindex < NUPDE) { /* We just released a PT, unhold the matching PD */ vm_page_t pdpg; @@ -1613,6 +1629,7 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK, ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK")); + PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * Allocate a page table page. */ @@ -1730,7 +1747,7 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t *pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; } - pmap->pm_stats.resident_count++; + pmap_resident_count_inc(pmap, 1); return (m); } @@ -2037,7 +2054,7 @@ pmap_collect(pmap_t locked_pmap, struct PMAP_LOCK(pmap); else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) continue; - pmap->pm_stats.resident_count--; + pmap_resident_count_dec(pmap, 1); pde = pmap_pde(pmap, va); KASSERT((*pde & PG_PS) == 0, ("pmap_collect: found" " a 2mpage in page %p's pv list", m)); @@ -2438,7 +2455,7 @@ pmap_demote_pde(pmap_t pmap, pd_entry_t return (FALSE); } if (va < VM_MAXUSER_ADDRESS) - pmap->pm_stats.resident_count++; + pmap_resident_count_inc(pmap, 1); } mptepa = VM_PAGE_TO_PHYS(mpte); firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa); @@ -2530,7 +2547,7 @@ pmap_remove_pde(pmap_t pmap, pd_entry_t */ if (oldpde & PG_G) pmap_invalidate_page(kernel_pmap, sva); - pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; + pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE); if (oldpde & PG_MANAGED) { pvh = pa_to_pvh(oldpde & PG_PS_FRAME); pmap_pvh_free(pvh, pmap, sva); @@ -2553,7 +2570,7 @@ pmap_remove_pde(pmap_t pmap, pd_entry_t mpte = pmap_lookup_pt_page(pmap, sva); if (mpte != NULL) { pmap_remove_pt_page(pmap, mpte); - pmap->pm_stats.resident_count--; + pmap_resident_count_dec(pmap, 1); KASSERT(mpte->wire_count == NPTEPG, ("pmap_remove_pde: pte page wire count error")); mpte->wire_count = 0; @@ -2584,7 +2601,7 @@ pmap_remove_pte(pmap_t pmap, pt_entry_t */ if (oldpte & PG_G) pmap_invalidate_page(kernel_pmap, va); - pmap->pm_stats.resident_count -= 1; + pmap_resident_count_dec(pmap, 1); if (oldpte & PG_MANAGED) { m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME); if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) @@ -2786,7 +2803,7 @@ pmap_remove_all(vm_page_t m) while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); - pmap->pm_stats.resident_count--; + pmap_resident_count_dec(pmap, 1); pde = pmap_pde(pmap, pv->pv_va); KASSERT((*pde & PG_PS) == 0, ("pmap_remove_all: found" " a 2mpage in page %p's pv list", m)); @@ -3192,7 +3209,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va, " va: 0x%lx", va)); } } else - pmap->pm_stats.resident_count++; + pmap_resident_count_inc(pmap, 1); /* * Enter on the PV list if part of our managed memory. @@ -3326,7 +3343,7 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t /* * Increment counters. */ - pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE; + pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE); /* * Map the superpage. @@ -3480,7 +3497,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_ /* * Increment counters */ - pmap->pm_stats.resident_count++; + pmap_resident_count_inc(pmap, 1); pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.pat_mode, 0); if ((prot & VM_PROT_EXECUTE) == 0) @@ -3585,8 +3602,7 @@ pmap_object_init_pt(pmap_t pmap, vm_offs if ((*pde & PG_V) == 0) { pde_store(pde, pa | PG_PS | PG_M | PG_A | PG_U | PG_RW | PG_V); - pmap->pm_stats.resident_count += NBPDR / - PAGE_SIZE; + pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE); pmap_pde_mappings++; } else { /* Continue on if the PDE is already valid. */ @@ -3653,8 +3669,6 @@ out: PMAP_UNLOCK(pmap); } - - /* * Copy the range specified by src_addr/len * from the source map to the range dst_addr/len @@ -3729,8 +3743,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pm pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr & PG_PS_FRAME))) { *pde = srcptepaddr & ~PG_W; - dst_pmap->pm_stats.resident_count += - NBPDR / PAGE_SIZE; + pmap_resident_count_inc(dst_pmap, NBPDR / PAGE_SIZE); } else dstmpde->wire_count--; continue; @@ -3773,7 +3786,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pm */ *dst_pte = ptetemp & ~(PG_W | PG_M | PG_A); - dst_pmap->pm_stats.resident_count++; + pmap_resident_count_inc(dst_pmap, 1); } else { free = NULL; if (pmap_unwire_pte_hold(dst_pmap, @@ -4046,7 +4059,7 @@ pmap_remove_pages(pmap_t pmap) pv_entry_count--; pc->pc_map[field] |= bitmask; if ((tpte & PG_PS) != 0) { - pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; + pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE); pvh = pa_to_pvh(tpte & PG_PS_FRAME); TAILQ_REMOVE(&pvh->pv_list, pv, pv_list); if (TAILQ_EMPTY(&pvh->pv_list)) { @@ -4057,7 +4070,7 @@ pmap_remove_pages(pmap_t pmap) mpte = pmap_lookup_pt_page(pmap, pv->pv_va); if (mpte != NULL) { pmap_remove_pt_page(pmap, mpte); - pmap->pm_stats.resident_count--; + pmap_resident_count_dec(pmap, 1); KASSERT(mpte->wire_count == NPTEPG, ("pmap_remove_pages: pte page wire count error")); mpte->wire_count = 0; @@ -4065,7 +4078,7 @@ pmap_remove_pages(pmap_t pmap) atomic_subtract_int(&cnt.v_wire_count, 1); } } else { - pmap->pm_stats.resident_count--; + pmap_resident_count_dec(pmap, 1); TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); if (TAILQ_EMPTY(&m->md.pv_list)) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); @@ -4223,9 +4236,9 @@ pmap_remove_write(vm_page_t m) mtx_assert(&vm_page_queue_mtx, MA_OWNED); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) { - va = pv->pv_va; pmap = PV_PMAP(pv); PMAP_LOCK(pmap); + va = pv->pv_va; pde = pmap_pde(pmap, va); if ((*pde & PG_RW) != 0) (void)pmap_demote_pde(pmap, pde, va); @@ -4281,9 +4294,9 @@ pmap_ts_referenced(vm_page_t m) mtx_assert(&vm_page_queue_mtx, MA_OWNED); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, pvn) { - va = pv->pv_va; pmap = PV_PMAP(pv); PMAP_LOCK(pmap); + va = pv->pv_va; pde = pmap_pde(pmap, va); oldpde = *pde; if ((oldpde & PG_A) != 0) { @@ -4353,9 +4366,9 @@ pmap_clear_modify(vm_page_t m) mtx_assert(&vm_page_queue_mtx, MA_OWNED); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) { - va = pv->pv_va; pmap = PV_PMAP(pv); PMAP_LOCK(pmap); + va = pv->pv_va; pde = pmap_pde(pmap, va); oldpde = *pde; if ((oldpde & PG_RW) != 0) { @@ -4418,9 +4431,9 @@ pmap_clear_reference(vm_page_t m) mtx_assert(&vm_page_queue_mtx, MA_OWNED); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) { - va = pv->pv_va; pmap = PV_PMAP(pv); PMAP_LOCK(pmap); + va = pv->pv_va; pde = pmap_pde(pmap, va); oldpde = *pde; if ((oldpde & PG_A) != 0) {