Date: Wed, 7 Aug 2013 16:33:15 +0000 (UTC) From: Konstantin Belousov <kib@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r254064 - in head/sys/amd64: amd64 include Message-ID: <201308071633.r77GXF0T092188@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kib Date: Wed Aug 7 16:33:15 2013 New Revision: 254064 URL: http://svnweb.freebsd.org/changeset/base/254064 Log: Change the pmap_ts_referenced() method of amd64 pmap to use shared pvh_global_lock. This allows the method to be executed in parallel, avoiding undue contention on the pvh_global_lock for the multithreaded pagedaemon. The pmap_ts_referenced() function has to inspect the page mappings for several pmaps, which need to be locked while pv list lock is owned. This contradicts to the lock order, where pmap lock is before pv list lock. Introduce the generation count for the pv list of the page or superpage, which indicate any change in the pv list, and, as usual, perform restart of the iteration if generation changed while pv lock was dropped for blocking acquire of a pmap lock. Reported and tested by: pho Reviewed by: alc Sponsored by: The FreeBSD Foundation Modified: head/sys/amd64/amd64/pmap.c head/sys/amd64/include/pmap.h Modified: head/sys/amd64/amd64/pmap.c ============================================================================== --- head/sys/amd64/amd64/pmap.c Wed Aug 7 16:01:45 2013 (r254063) +++ head/sys/amd64/amd64/pmap.c Wed Aug 7 16:33:15 2013 (r254064) @@ -2183,6 +2183,7 @@ reclaim_pv_chunk(pmap_t locked_pmap, str vm_page_aflag_set(m, PGA_REFERENCED); CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); @@ -2470,6 +2471,7 @@ pmap_pvh_remove(struct md_page *pvh, pma TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { if (pmap == PV_PMAP(pv) && va == pv->pv_va) { TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); + pvh->pv_gen++; break; } } @@ -2509,6 +2511,7 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offse KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found")); m = PHYS_TO_VM_PAGE(pa); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; /* Instantiate the remaining NPTEPG - 1 pv entries. */ PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1)); va_last = va + NBPDR - PAGE_SIZE; @@ -2527,6 +2530,7 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offse KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_pv_demote_pde: page %p is not managed", m)); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; if (va == va_last) goto out; } @@ -2575,6 +2579,7 @@ pmap_pv_promote_pde(pmap_t pmap, vm_offs KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found")); pvh = pa_to_pvh(pa); TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); + pvh->pv_gen++; /* Free the remaining NPTEPG - 1 pv entries. */ va_last = va + NBPDR - PAGE_SIZE; do { @@ -2616,6 +2621,7 @@ pmap_try_insert_pv_entry(pmap_t pmap, vm pv->pv_va = va; CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; return (TRUE); } else return (FALSE); @@ -2640,6 +2646,7 @@ pmap_pv_insert_pde(pmap_t pmap, vm_offse CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); pvh = pa_to_pvh(pa); TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); + pvh->pv_gen++; return (TRUE); } else return (FALSE); @@ -3156,6 +3163,7 @@ small_mappings: pmap_unuse_pt(pmap, pv->pv_va, *pde, &free); pmap_invalidate_page(pmap, pv->pv_va); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; free_pv_entry(pmap, pv); PMAP_UNLOCK(pmap); } @@ -3601,6 +3609,7 @@ retry: pv->pv_va = va; CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; if ((newpte & PG_RW) != 0) vm_page_aflag_set(m, PGA_WRITEABLE); } @@ -4536,6 +4545,7 @@ pmap_remove_pages(pmap_t pmap) pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE); pvh = pa_to_pvh(tpte & PG_PS_FRAME); TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); + pvh->pv_gen++; if (TAILQ_EMPTY(&pvh->pv_list)) { for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++) if ((mt->aflags & PGA_WRITEABLE) != 0 && @@ -4555,6 +4565,7 @@ pmap_remove_pages(pmap_t pmap) } else { pmap_resident_count_dec(pmap, 1); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; if ((m->aflags & PGA_WRITEABLE) != 0 && TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { @@ -4793,25 +4804,40 @@ pmap_ts_referenced(vm_page_t m) struct md_page *pvh; pv_entry_t pv, pvf, pvn; pmap_t pmap; + struct rwlock *lock; pd_entry_t oldpde, *pde; pt_entry_t *pte; vm_offset_t va; - int rtval = 0; + int rtval, pvh_gen, md_gen; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_ts_referenced: page %p is not managed", m)); - rw_wlock(&pvh_global_lock); + rw_rlock(&pvh_global_lock); + lock = VM_PAGE_TO_PV_LIST_LOCK(m); + pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); + rtval = 0; +retry: + rw_wlock(lock); if ((m->flags & PG_FICTITIOUS) != 0) goto small_mappings; - pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, pvn) { pmap = PV_PMAP(pv); - PMAP_LOCK(pmap); + if (!PMAP_TRYLOCK(pmap)) { + pvh_gen = pvh->pv_gen; + rw_wunlock(lock); + PMAP_LOCK(pmap); + rw_wlock(lock); + if (pvh_gen != pvh->pv_gen) { + PMAP_UNLOCK(pmap); + rw_wunlock(lock); + goto retry; + } + } va = pv->pv_va; pde = pmap_pde(pmap, va); oldpde = *pde; if ((oldpde & PG_A) != 0) { - if (pmap_demote_pde(pmap, pde, va)) { + if (pmap_demote_pde_locked(pmap, pde, va, &lock)) { if ((oldpde & PG_W) == 0) { /* * Remove the mapping to a single page @@ -4823,7 +4849,10 @@ pmap_ts_referenced(vm_page_t m) */ va += VM_PAGE_TO_PHYS(m) - (oldpde & PG_PS_FRAME); - pmap_remove_page(pmap, va, pde, NULL); + pte = pmap_pde_to_pte(pde, va); + pmap_remove_pte(pmap, pte, va, *pde, + NULL, &lock); + pmap_invalidate_page(pmap, va); rtval++; if (rtval > 4) { PMAP_UNLOCK(pmap); @@ -4831,6 +4860,9 @@ pmap_ts_referenced(vm_page_t m) } } } + KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), + ("inconsistent pv lock %p %p for page %p", + lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); } PMAP_UNLOCK(pmap); } @@ -4841,8 +4873,21 @@ small_mappings: pvn = TAILQ_NEXT(pv, pv_next); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; pmap = PV_PMAP(pv); - PMAP_LOCK(pmap); + if (!PMAP_TRYLOCK(pmap)) { + pvh_gen = pvh->pv_gen; + md_gen = m->md.pv_gen; + rw_wunlock(lock); + PMAP_LOCK(pmap); + rw_wlock(lock); + if (pvh_gen != pvh->pv_gen || + md_gen != m->md.pv_gen) { + PMAP_UNLOCK(pmap); + rw_wunlock(lock); + goto retry; + } + } pde = pmap_pde(pmap, pv->pv_va); KASSERT((*pde & PG_PS) == 0, ("pmap_ts_referenced:" " found a 2mpage in page %p's pv list", m)); @@ -4858,7 +4903,8 @@ small_mappings: } while ((pv = pvn) != NULL && pv != pvf); } out: - rw_wunlock(&pvh_global_lock); + rw_wunlock(lock); + rw_runlock(&pvh_global_lock); return (rtval); } Modified: head/sys/amd64/include/pmap.h ============================================================================== --- head/sys/amd64/include/pmap.h Wed Aug 7 16:01:45 2013 (r254063) +++ head/sys/amd64/include/pmap.h Wed Aug 7 16:33:15 2013 (r254064) @@ -233,6 +233,7 @@ struct pv_chunk; struct md_page { TAILQ_HEAD(,pv_entry) pv_list; + int pv_gen; int pat_mode; };
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201308071633.r77GXF0T092188>