Date: Thu, 10 Jun 2010 16:56:35 +0000 (UTC) From: Alan Cox <alc@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r208990 - in head/sys: amd64/amd64 arm/arm i386/i386 i386/xen ia64/ia64 mips/mips powerpc/aim powerpc/booke sparc64/sparc64 sun4v/sun4v vm Message-ID: <201006101656.o5AGuZuO009002@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: alc Date: Thu Jun 10 16:56:35 2010 New Revision: 208990 URL: http://svn.freebsd.org/changeset/base/208990 Log: Reduce the scope of the page queues lock and the number of PG_REFERENCED changes in vm_pageout_object_deactivate_pages(). Simplify this function's inner loop using TAILQ_FOREACH(), and shorten some of its overly long lines. Update a stale comment. Assert that PG_REFERENCED may be cleared only if the object containing the page is locked. Add a comment documenting this. Assert that a caller to vm_page_requeue() holds the page queues lock, and assert that the page is on a page queue. Push down the page queues lock into pmap_ts_referenced() and pmap_page_exists_quick(). (As of now, there are no longer any pmap functions that expect to be called with the page queues lock held.) Neither pmap_ts_referenced() nor pmap_page_exists_quick() should ever be passed an unmanaged page. Assert this rather than returning "0" and "FALSE" respectively. ARM: Simplify pmap_page_exists_quick() by switching to TAILQ_FOREACH(). Push down the page queues lock inside of pmap_clearbit(), simplifying pmap_clear_modify(), pmap_clear_reference(), and pmap_remove_write(). Additionally, this allows for avoiding the acquisition of the page queues lock in some cases. PowerPC/AIM: moea*_page_exits_quick() and moea*_page_wired_mappings() will never be called before pmap initialization is complete. Therefore, the check for moea_initialized can be eliminated. Push down the page queues lock inside of moea*_clear_bit(), simplifying moea*_clear_modify() and moea*_clear_reference(). The last parameter to moea*_clear_bit() is never used. Eliminate it. PowerPC/BookE: Simplify mmu_booke_page_exists_quick()'s control flow. Reviewed by: kib@ Modified: head/sys/amd64/amd64/pmap.c head/sys/arm/arm/pmap.c head/sys/i386/i386/pmap.c head/sys/i386/xen/pmap.c head/sys/ia64/ia64/pmap.c head/sys/mips/mips/pmap.c head/sys/powerpc/aim/mmu_oea.c head/sys/powerpc/aim/mmu_oea64.c head/sys/powerpc/booke/pmap.c head/sys/sparc64/sparc64/pmap.c head/sys/sun4v/sun4v/pmap.c head/sys/vm/vm_page.c head/sys/vm/vm_page.h head/sys/vm/vm_pageout.c Modified: head/sys/amd64/amd64/pmap.c ============================================================================== --- head/sys/amd64/amd64/pmap.c Thu Jun 10 16:45:30 2010 (r208989) +++ head/sys/amd64/amd64/pmap.c Thu Jun 10 16:56:35 2010 (r208990) @@ -3899,30 +3899,35 @@ pmap_page_exists_quick(pmap_t pmap, vm_p struct md_page *pvh; pv_entry_t pv; int loops = 0; + boolean_t rv; - if (m->flags & PG_FICTITIOUS) - return (FALSE); - - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_page_exists_quick: page %p is not managed", m)); + rv = FALSE; + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (PV_PMAP(pv) == pmap) { - return (TRUE); + rv = TRUE; + break; } loops++; if (loops >= 16) break; } - if (loops < 16) { + if (!rv && loops < 16) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) { - if (PV_PMAP(pv) == pmap) - return (TRUE); + if (PV_PMAP(pv) == pmap) { + rv = TRUE; + break; + } loops++; if (loops >= 16) break; } } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -4335,10 +4340,10 @@ pmap_ts_referenced(vm_page_t m) vm_offset_t va; int rtval = 0; - if (m->flags & PG_FICTITIOUS) - return (rtval); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_ts_referenced: page %p is not managed", m)); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); + vm_page_lock_queues(); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, pvn) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); @@ -4362,7 +4367,7 @@ pmap_ts_referenced(vm_page_t m) rtval++; if (rtval > 4) { PMAP_UNLOCK(pmap); - return (rtval); + goto out; } } } @@ -4391,6 +4396,8 @@ pmap_ts_referenced(vm_page_t m) PMAP_UNLOCK(pmap); } while ((pv = pvn) != NULL && pv != pvf); } +out: + vm_page_unlock_queues(); return (rtval); } Modified: head/sys/arm/arm/pmap.c ============================================================================== --- head/sys/arm/arm/pmap.c Thu Jun 10 16:45:30 2010 (r208989) +++ head/sys/arm/arm/pmap.c Thu Jun 10 16:56:35 2010 (r208990) @@ -1423,7 +1423,7 @@ pmap_clearbit(struct vm_page *pg, u_int u_int oflags; int count = 0; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + vm_page_lock_queues(); if (maskbits & PVF_WRITE) maskbits |= PVF_MOD; @@ -1433,6 +1433,7 @@ pmap_clearbit(struct vm_page *pg, u_int pg->md.pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF)); if (TAILQ_EMPTY(&pg->md.pv_list)) { + vm_page_unlock_queues(); return (0); } @@ -1568,6 +1569,7 @@ pmap_clearbit(struct vm_page *pg, u_int if (maskbits & PVF_WRITE) vm_page_flag_clear(pg, PG_WRITEABLE); + vm_page_unlock_queues(); return (count); } @@ -4417,24 +4419,23 @@ pmap_page_exists_quick(pmap_t pmap, vm_p { pv_entry_t pv; int loops = 0; + boolean_t rv; - if (m->flags & PG_FICTITIOUS) - return (FALSE); - - /* - * Not found, check current mappings returning immediately - */ - for (pv = TAILQ_FIRST(&m->md.pv_list); - pv; - pv = TAILQ_NEXT(pv, pv_list)) { + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_page_exists_quick: page %p is not managed", m)); + rv = FALSE; + vm_page_lock_queues(); + TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (pv->pv_pmap == pmap) { - return (TRUE); + rv = TRUE; + break; } loops++; if (loops >= 16) break; } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -4469,8 +4470,8 @@ int pmap_ts_referenced(vm_page_t m) { - if (m->flags & PG_FICTITIOUS) - return (0); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_ts_referenced: page %p is not managed", m)); return (pmap_clearbit(m, PVF_REF)); } @@ -4508,10 +4509,8 @@ pmap_clear_modify(vm_page_t m) */ if ((m->flags & PG_WRITEABLE) == 0) return; - vm_page_lock_queues(); if (m->md.pvh_attrs & PVF_MOD) pmap_clearbit(m, PVF_MOD); - vm_page_unlock_queues(); } @@ -4541,10 +4540,8 @@ pmap_clear_reference(vm_page_t m) KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, ("pmap_clear_reference: page %p is not managed", m)); - vm_page_lock_queues(); if (m->md.pvh_attrs & PVF_REF) pmap_clearbit(m, PVF_REF); - vm_page_unlock_queues(); } @@ -4565,11 +4562,8 @@ pmap_remove_write(vm_page_t m) */ VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); if ((m->oflags & VPO_BUSY) != 0 || - (m->flags & PG_WRITEABLE) != 0) { - vm_page_lock_queues(); + (m->flags & PG_WRITEABLE) != 0) pmap_clearbit(m, PVF_WRITE); - vm_page_unlock_queues(); - } } Modified: head/sys/i386/i386/pmap.c ============================================================================== --- head/sys/i386/i386/pmap.c Thu Jun 10 16:45:30 2010 (r208989) +++ head/sys/i386/i386/pmap.c Thu Jun 10 16:56:35 2010 (r208990) @@ -4061,30 +4061,35 @@ pmap_page_exists_quick(pmap_t pmap, vm_p struct md_page *pvh; pv_entry_t pv; int loops = 0; + boolean_t rv; - if (m->flags & PG_FICTITIOUS) - return (FALSE); - - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_page_exists_quick: page %p is not managed", m)); + rv = FALSE; + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (PV_PMAP(pv) == pmap) { - return (TRUE); + rv = TRUE; + break; } loops++; if (loops >= 16) break; } - if (loops < 16) { + if (!rv && loops < 16) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) { - if (PV_PMAP(pv) == pmap) - return (TRUE); + if (PV_PMAP(pv) == pmap) { + rv = TRUE; + break; + } loops++; if (loops >= 16) break; } } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -4512,11 +4517,11 @@ pmap_ts_referenced(vm_page_t m) vm_offset_t va; int rtval = 0; - if (m->flags & PG_FICTITIOUS) - return (rtval); - sched_pin(); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_ts_referenced: page %p is not managed", m)); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); + vm_page_lock_queues(); + sched_pin(); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, pvn) { va = pv->pv_va; pmap = PV_PMAP(pv); @@ -4571,6 +4576,7 @@ pmap_ts_referenced(vm_page_t m) } out: sched_unpin(); + vm_page_unlock_queues(); return (rtval); } Modified: head/sys/i386/xen/pmap.c ============================================================================== --- head/sys/i386/xen/pmap.c Thu Jun 10 16:45:30 2010 (r208989) +++ head/sys/i386/xen/pmap.c Thu Jun 10 16:56:35 2010 (r208990) @@ -3449,20 +3449,23 @@ pmap_page_exists_quick(pmap_t pmap, vm_p { pv_entry_t pv; int loops = 0; + boolean_t rv; - if (m->flags & PG_FICTITIOUS) - return (FALSE); - - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_page_exists_quick: page %p is not managed", m)); + rv = FALSE; + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (PV_PMAP(pv) == pmap) { - return TRUE; + rv = TRUE; + break; } loops++; if (loops >= 16) break; } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -3839,10 +3842,10 @@ pmap_ts_referenced(vm_page_t m) pt_entry_t *pte; int rtval = 0; - if (m->flags & PG_FICTITIOUS) - return (rtval); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_ts_referenced: page %p is not managed", m)); + vm_page_lock_queues(); sched_pin(); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { pvf = pv; do { @@ -3867,6 +3870,7 @@ pmap_ts_referenced(vm_page_t m) PT_SET_MA(PADDR1, 0); sched_unpin(); + vm_page_unlock_queues(); return (rtval); } Modified: head/sys/ia64/ia64/pmap.c ============================================================================== --- head/sys/ia64/ia64/pmap.c Thu Jun 10 16:45:30 2010 (r208989) +++ head/sys/ia64/ia64/pmap.c Thu Jun 10 16:56:35 2010 (r208990) @@ -1837,23 +1837,23 @@ pmap_page_exists_quick(pmap_t pmap, vm_p { pv_entry_t pv; int loops = 0; + boolean_t rv; - if (m->flags & PG_FICTITIOUS) - return FALSE; - - /* - * Not found, check current mappings returning immediately if found. - */ - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_page_exists_quick: page %p is not managed", m)); + rv = FALSE; + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (pv->pv_pmap == pmap) { - return TRUE; + rv = TRUE; + break; } loops++; if (loops >= 16) break; } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -1949,9 +1949,9 @@ pmap_ts_referenced(vm_page_t m) pv_entry_t pv; int count = 0; - if (m->flags & PG_FICTITIOUS) - return 0; - + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_ts_referenced: page %p is not managed", m)); + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { PMAP_LOCK(pv->pv_pmap); oldpmap = pmap_switch(pv->pv_pmap); @@ -1965,8 +1965,8 @@ pmap_ts_referenced(vm_page_t m) pmap_switch(oldpmap); PMAP_UNLOCK(pv->pv_pmap); } - - return count; + vm_page_unlock_queues(); + return (count); } /* Modified: head/sys/mips/mips/pmap.c ============================================================================== --- head/sys/mips/mips/pmap.c Thu Jun 10 16:45:30 2010 (r208989) +++ head/sys/mips/mips/pmap.c Thu Jun 10 16:56:35 2010 (r208990) @@ -2348,20 +2348,23 @@ pmap_page_exists_quick(pmap_t pmap, vm_p { pv_entry_t pv; int loops = 0; + boolean_t rv; - if (m->flags & PG_FICTITIOUS) - return FALSE; - - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_page_exists_quick: page %p is not managed", m)); + rv = FALSE; + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (pv->pv_pmap == pmap) { - return TRUE; + rv = TRUE; + break; } loops++; if (loops >= 16) break; } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -2594,14 +2597,16 @@ pmap_remove_write(vm_page_t m) int pmap_ts_referenced(vm_page_t m) { - if (m->flags & PG_FICTITIOUS) - return (0); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_ts_referenced: page %p is not managed", m)); if (m->md.pv_flags & PV_TABLE_REF) { + vm_page_lock_queues(); m->md.pv_flags &= ~PV_TABLE_REF; - return 1; + vm_page_unlock_queues(); + return (1); } - return 0; + return (0); } /* Modified: head/sys/powerpc/aim/mmu_oea.c ============================================================================== --- head/sys/powerpc/aim/mmu_oea.c Thu Jun 10 16:45:30 2010 (r208989) +++ head/sys/powerpc/aim/mmu_oea.c Thu Jun 10 16:56:35 2010 (r208990) @@ -286,7 +286,7 @@ static void moea_enter_locked(pmap_t, v vm_prot_t, boolean_t); static void moea_syncicache(vm_offset_t, vm_size_t); static boolean_t moea_query_bit(vm_page_t, int); -static u_int moea_clear_bit(vm_page_t, int, int *); +static u_int moea_clear_bit(vm_page_t, int); static void moea_kremove(mmu_t, vm_offset_t); int moea_pte_spill(vm_offset_t); @@ -1315,9 +1315,7 @@ moea_clear_reference(mmu_t mmu, vm_page_ KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, ("moea_clear_reference: page %p is not managed", m)); - vm_page_lock_queues(); - moea_clear_bit(m, PTE_REF, NULL); - vm_page_unlock_queues(); + moea_clear_bit(m, PTE_REF); } void @@ -1337,9 +1335,7 @@ moea_clear_modify(mmu_t mmu, vm_page_t m */ if ((m->flags & PG_WRITEABLE) == 0) return; - vm_page_lock_queues(); - moea_clear_bit(m, PTE_CHG, NULL); - vm_page_unlock_queues(); + moea_clear_bit(m, PTE_CHG); } /* @@ -1409,14 +1405,10 @@ moea_remove_write(mmu_t mmu, vm_page_t m boolean_t moea_ts_referenced(mmu_t mmu, vm_page_t m) { - int count; - - if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) - return (0); - count = moea_clear_bit(m, PTE_REF, NULL); - - return (count); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("moea_ts_referenced: page %p is not managed", m)); + return (moea_clear_bit(m, PTE_REF)); } /* @@ -1531,19 +1523,23 @@ moea_page_exists_quick(mmu_t mmu, pmap_t { int loops; struct pvo_entry *pvo; + boolean_t rv; - if (!moea_initialized || (m->flags & PG_FICTITIOUS)) - return FALSE; - + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("moea_page_exists_quick: page %p is not managed", m)); loops = 0; + rv = FALSE; + vm_page_lock_queues(); LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { - if (pvo->pvo_pmap == pmap) - return (TRUE); + if (pvo->pvo_pmap == pmap) { + rv = TRUE; + break; + } if (++loops >= 16) break; } - - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -1557,7 +1553,7 @@ moea_page_wired_mappings(mmu_t mmu, vm_p int count; count = 0; - if (!moea_initialized || (m->flags & PG_FICTITIOUS) != 0) + if ((m->flags & PG_FICTITIOUS) != 0) return (count); vm_page_lock_queues(); LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) @@ -2315,17 +2311,17 @@ moea_query_bit(vm_page_t m, int ptebit) } static u_int -moea_clear_bit(vm_page_t m, int ptebit, int *origbit) +moea_clear_bit(vm_page_t m, int ptebit) { u_int count; struct pvo_entry *pvo; struct pte *pt; - int rv; + + vm_page_lock_queues(); /* * Clear the cached value. */ - rv = moea_attr_fetch(m); moea_attr_clear(m, ptebit); /* @@ -2353,15 +2349,11 @@ moea_clear_bit(vm_page_t m, int ptebit, } mtx_unlock(&moea_table_mutex); } - rv |= pvo->pvo_pte.pte.pte_lo; pvo->pvo_pte.pte.pte_lo &= ~ptebit; MOEA_PVO_CHECK(pvo); /* sanity check */ } - if (origbit != NULL) { - *origbit = rv; - } - + vm_page_unlock_queues(); return (count); } Modified: head/sys/powerpc/aim/mmu_oea64.c ============================================================================== --- head/sys/powerpc/aim/mmu_oea64.c Thu Jun 10 16:45:30 2010 (r208989) +++ head/sys/powerpc/aim/mmu_oea64.c Thu Jun 10 16:56:35 2010 (r208990) @@ -358,7 +358,7 @@ static void moea64_bridge_cpu_bootstrap static void moea64_enter_locked(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, boolean_t); static boolean_t moea64_query_bit(vm_page_t, u_int64_t); -static u_int moea64_clear_bit(vm_page_t, u_int64_t, u_int64_t *); +static u_int moea64_clear_bit(vm_page_t, u_int64_t); static void moea64_kremove(mmu_t, vm_offset_t); static void moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_offset_t pa, vm_size_t sz); @@ -1510,9 +1510,7 @@ moea64_clear_reference(mmu_t mmu, vm_pag KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, ("moea64_clear_reference: page %p is not managed", m)); - vm_page_lock_queues(); - moea64_clear_bit(m, LPTE_REF, NULL); - vm_page_unlock_queues(); + moea64_clear_bit(m, LPTE_REF); } void @@ -1532,9 +1530,7 @@ moea64_clear_modify(mmu_t mmu, vm_page_t */ if ((m->flags & PG_WRITEABLE) == 0) return; - vm_page_lock_queues(); - moea64_clear_bit(m, LPTE_CHG, NULL); - vm_page_unlock_queues(); + moea64_clear_bit(m, LPTE_CHG); } /* @@ -1605,14 +1601,10 @@ moea64_remove_write(mmu_t mmu, vm_page_t boolean_t moea64_ts_referenced(mmu_t mmu, vm_page_t m) { - int count; - - if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) - return (0); - count = moea64_clear_bit(m, LPTE_REF, NULL); - - return (count); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("moea64_ts_referenced: page %p is not managed", m)); + return (moea64_clear_bit(m, LPTE_REF)); } /* @@ -1721,21 +1713,23 @@ moea64_page_exists_quick(mmu_t mmu, pmap { int loops; struct pvo_entry *pvo; + boolean_t rv; - if (!moea64_initialized || (m->flags & PG_FICTITIOUS)) - return FALSE; - - mtx_assert(&vm_page_queue_mtx, MA_OWNED); - + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("moea64_page_exists_quick: page %p is not managed", m)); loops = 0; + rv = FALSE; + vm_page_lock_queues(); LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { - if (pvo->pvo_pmap == pmap) - return (TRUE); + if (pvo->pvo_pmap == pmap) { + rv = TRUE; + break; + } if (++loops >= 16) break; } - - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -1749,7 +1743,7 @@ moea64_page_wired_mappings(mmu_t mmu, vm int count; count = 0; - if (!moea64_initialized || (m->flags & PG_FICTITIOUS) != 0) + if ((m->flags & PG_FICTITIOUS) != 0) return (count); vm_page_lock_queues(); LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) @@ -2445,19 +2439,17 @@ moea64_query_bit(vm_page_t m, u_int64_t } static u_int -moea64_clear_bit(vm_page_t m, u_int64_t ptebit, u_int64_t *origbit) +moea64_clear_bit(vm_page_t m, u_int64_t ptebit) { u_int count; struct pvo_entry *pvo; struct lpte *pt; - uint64_t rv; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + vm_page_lock_queues(); /* * Clear the cached value. */ - rv = moea64_attr_fetch(m); moea64_attr_clear(m, ptebit); /* @@ -2486,16 +2478,12 @@ moea64_clear_bit(vm_page_t m, u_int64_t moea64_pte_clear(pt, pvo->pvo_pmap, PVO_VADDR(pvo), ptebit); } } - rv |= pvo->pvo_pte.lpte.pte_lo; pvo->pvo_pte.lpte.pte_lo &= ~ptebit; MOEA_PVO_CHECK(pvo); /* sanity check */ UNLOCK_TABLE(); } - if (origbit != NULL) { - *origbit = rv; - } - + vm_page_unlock_queues(); return (count); } Modified: head/sys/powerpc/booke/pmap.c ============================================================================== --- head/sys/powerpc/booke/pmap.c Thu Jun 10 16:45:30 2010 (r208989) +++ head/sys/powerpc/booke/pmap.c Thu Jun 10 16:56:35 2010 (r208990) @@ -2293,17 +2293,14 @@ mmu_booke_ts_referenced(mmu_t mmu, vm_pa pv_entry_t pv; int count; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); - if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) - return (0); - + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("mmu_booke_ts_referenced: page %p is not managed", m)); count = 0; + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { PMAP_LOCK(pv->pv_pmap); - if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL) { - if (!PTE_ISVALID(pte)) - goto make_sure_to_unlock; - + if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL && + PTE_ISVALID(pte)) { if (PTE_ISREFERENCED(pte)) { mtx_lock_spin(&tlbivax_mutex); tlb_miss_lock(); @@ -2320,9 +2317,9 @@ mmu_booke_ts_referenced(mmu_t mmu, vm_pa } } } -make_sure_to_unlock: PMAP_UNLOCK(pv->pv_pmap); } + vm_page_unlock_queues(); return (count); } @@ -2394,20 +2391,23 @@ mmu_booke_page_exists_quick(mmu_t mmu, p { pv_entry_t pv; int loops; + boolean_t rv; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); - if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) - return (FALSE); - + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("mmu_booke_page_exists_quick: page %p is not managed", m)); loops = 0; + rv = FALSE; + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { - if (pv->pv_pmap == pmap) - return (TRUE); - + if (pv->pv_pmap == pmap) { + rv = TRUE; + break; + } if (++loops >= 16) break; } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* Modified: head/sys/sparc64/sparc64/pmap.c ============================================================================== --- head/sys/sparc64/sparc64/pmap.c Thu Jun 10 16:45:30 2010 (r208989) +++ head/sys/sparc64/sparc64/pmap.c Thu Jun 10 16:56:35 2010 (r208990) @@ -1789,20 +1789,25 @@ pmap_page_exists_quick(pmap_t pm, vm_pag { struct tte *tp; int loops; + boolean_t rv; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); - if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) - return (FALSE); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_page_exists_quick: page %p is not managed", m)); loops = 0; + rv = FALSE; + vm_page_lock_queues(); TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; - if (TTE_GET_PMAP(tp) == pm) - return (TRUE); + if (TTE_GET_PMAP(tp) == pm) { + rv = TRUE; + break; + } if (++loops >= 16) break; } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -1878,10 +1883,10 @@ pmap_ts_referenced(vm_page_t m) u_long data; int count; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); - if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) - return (0); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_ts_referenced: page %p is not managed", m)); count = 0; + vm_page_lock_queues(); if ((tp = TAILQ_FIRST(&m->md.tte_list)) != NULL) { tpf = tp; do { @@ -1895,6 +1900,7 @@ pmap_ts_referenced(vm_page_t m) break; } while ((tp = tpn) != NULL && tp != tpf); } + vm_page_unlock_queues(); return (count); } Modified: head/sys/sun4v/sun4v/pmap.c ============================================================================== --- head/sys/sun4v/sun4v/pmap.c Thu Jun 10 16:45:30 2010 (r208989) +++ head/sys/sun4v/sun4v/pmap.c Thu Jun 10 16:56:35 2010 (r208990) @@ -1738,20 +1738,23 @@ pmap_page_exists_quick(pmap_t pmap, vm_p { pv_entry_t pv; int loops = 0; + boolean_t rv; - if (m->flags & PG_FICTITIOUS) - return FALSE; - - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_page_exists_quick: page %p is not managed", m)); + rv = FALSE; + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (pv->pv_pmap == pmap) { - return TRUE; + rv = TRUE; + break; } loops++; if (loops >= 16) break; } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -2309,17 +2312,15 @@ pmap_tte_hash_resize(pmap_t pmap) int pmap_ts_referenced(vm_page_t m) { - int rv; pv_entry_t pv, pvf, pvn; pmap_t pmap; tte_t otte_data; + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_ts_referenced: page %p is not managed", m)); rv = 0; - if (m->flags & PG_FICTITIOUS) - return (rv); - - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + vm_page_lock_queues(); if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { pvf = pv; @@ -2347,6 +2348,7 @@ pmap_ts_referenced(vm_page_t m) PMAP_UNLOCK(pmap); } while ((pv = pvn) != NULL && pv != pvf); } + vm_page_unlock_queues(); return (rv); } Modified: head/sys/vm/vm_page.c ============================================================================== --- head/sys/vm/vm_page.c Thu Jun 10 16:45:30 2010 (r208989) +++ head/sys/vm/vm_page.c Thu Jun 10 16:56:35 2010 (r208990) @@ -502,6 +502,8 @@ vm_page_flag_clear(vm_page_t m, unsigned { mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((bits & PG_REFERENCED) == 0 || VM_OBJECT_LOCKED(m->object), + ("PG_REFERENCED and !VM_OBJECT_LOCKED")); m->flags &= ~bits; } @@ -1333,8 +1335,7 @@ vm_waitpfault(void) /* * vm_page_requeue: * - * If the given page is contained within a page queue, move it to the tail - * of that queue. + * Move the given page to the tail of its present page queue. * * The page queues must be locked. */ @@ -1344,11 +1345,12 @@ vm_page_requeue(vm_page_t m) int queue = VM_PAGE_GETQUEUE(m); struct vpgqueues *vpq; - if (queue != PQ_NONE) { - vpq = &vm_page_queues[queue]; - TAILQ_REMOVE(&vpq->pl, m, pageq); - TAILQ_INSERT_TAIL(&vpq->pl, m, pageq); - } + mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT(queue != PQ_NONE, + ("vm_page_requeue: page %p is not queued", m)); + vpq = &vm_page_queues[queue]; + TAILQ_REMOVE(&vpq->pl, m, pageq); + TAILQ_INSERT_TAIL(&vpq->pl, m, pageq); } /* Modified: head/sys/vm/vm_page.h ============================================================================== --- head/sys/vm/vm_page.h Thu Jun 10 16:45:30 2010 (r208989) +++ head/sys/vm/vm_page.h Thu Jun 10 16:56:35 2010 (r208990) @@ -219,6 +219,9 @@ extern struct vpglocks pa_lock[]; * pte mappings, nor can they be removed from their objects via * the object, and such pages are also not on any PQ queue. * + * PG_REFERENCED may be cleared only if the object containing the page is + * locked. + * * PG_WRITEABLE is set exclusively on managed pages by pmap_enter(). When it * does so, the page must be VPO_BUSY. */ Modified: head/sys/vm/vm_pageout.c ============================================================================== --- head/sys/vm/vm_pageout.c Thu Jun 10 16:45:30 2010 (r208989) +++ head/sys/vm/vm_pageout.c Thu Jun 10 16:56:35 2010 (r208990) @@ -547,21 +547,17 @@ vm_pageout_flush(vm_page_t *mc, int coun /* * vm_pageout_object_deactivate_pages * - * deactivate enough pages to satisfy the inactive target - * requirements or if vm_page_proc_limit is set, then - * deactivate all of the pages in the object and its - * backing_objects. + * Deactivate enough pages to satisfy the inactive target + * requirements. * * The object and map must be locked. */ static void -vm_pageout_object_deactivate_pages(pmap, first_object, desired) - pmap_t pmap; - vm_object_t first_object; - long desired; +vm_pageout_object_deactivate_pages(pmap_t pmap, vm_object_t first_object, + long desired) { vm_object_t backing_object, object; - vm_page_t p, next; + vm_page_t p; int actcount, remove_mode; VM_OBJECT_LOCK_ASSERT(first_object, MA_OWNED); @@ -579,61 +575,57 @@ vm_pageout_object_deactivate_pages(pmap, if (object->shadow_count > 1) remove_mode = 1; /* - * scan the objects entire memory queue + * Scan the object's entire memory queue. */ - p = TAILQ_FIRST(&object->memq); - while (p != NULL) { + TAILQ_FOREACH(p, &object->memq, listq) { if (pmap_resident_count(pmap) <= desired) goto unlock_return; - next = TAILQ_NEXT(p, listq); - if ((p->oflags & VPO_BUSY) != 0 || p->busy != 0) { - p = next; + if ((p->oflags & VPO_BUSY) != 0 || p->busy != 0) continue; - } + PCPU_INC(cnt.v_pdpages); vm_page_lock(p); - vm_page_lock_queues(); - cnt.v_pdpages++; - if (p->wire_count != 0 || - p->hold_count != 0 || + if (p->wire_count != 0 || p->hold_count != 0 || !pmap_page_exists_quick(pmap, p)) { - vm_page_unlock_queues(); vm_page_unlock(p); - p = next; continue; } actcount = pmap_ts_referenced(p); - if (actcount) { - vm_page_flag_set(p, PG_REFERENCED); - } else if (p->flags & PG_REFERENCED) { - actcount = 1; + if ((p->flags & PG_REFERENCED) != 0) { + if (actcount == 0) + actcount = 1; + vm_page_lock_queues(); + vm_page_flag_clear(p, PG_REFERENCED); + vm_page_unlock_queues(); } - if ((p->queue != PQ_ACTIVE) && - (p->flags & PG_REFERENCED)) { + if (p->queue != PQ_ACTIVE && actcount != 0) { vm_page_activate(p); p->act_count += actcount; - vm_page_flag_clear(p, PG_REFERENCED); *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201006101656.o5AGuZuO009002>