From owner-svn-src-head@FreeBSD.ORG Wed Aug 7 16:33:16 2013 Return-Path: Delivered-To: svn-src-head@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTP id 5583E8D0; Wed, 7 Aug 2013 16:33:16 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mx1.freebsd.org (Postfix) with ESMTPS id 3456A2756; Wed, 7 Aug 2013 16:33:16 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.7/8.14.7) with ESMTP id r77GXG3K092191; Wed, 7 Aug 2013 16:33:16 GMT (envelope-from kib@svn.freebsd.org) Received: (from kib@localhost) by svn.freebsd.org (8.14.7/8.14.5/Submit) id r77GXF0T092188; Wed, 7 Aug 2013 16:33:15 GMT (envelope-from kib@svn.freebsd.org) Message-Id: <201308071633.r77GXF0T092188@svn.freebsd.org> From: Konstantin Belousov Date: Wed, 7 Aug 2013 16:33:15 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r254064 - in head/sys/amd64: amd64 include X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 07 Aug 2013 16:33:16 -0000 Author: kib Date: Wed Aug 7 16:33:15 2013 New Revision: 254064 URL: http://svnweb.freebsd.org/changeset/base/254064 Log: Change the pmap_ts_referenced() method of amd64 pmap to use shared pvh_global_lock. This allows the method to be executed in parallel, avoiding undue contention on the pvh_global_lock for the multithreaded pagedaemon. The pmap_ts_referenced() function has to inspect the page mappings for several pmaps, which need to be locked while pv list lock is owned. This contradicts to the lock order, where pmap lock is before pv list lock. Introduce the generation count for the pv list of the page or superpage, which indicate any change in the pv list, and, as usual, perform restart of the iteration if generation changed while pv lock was dropped for blocking acquire of a pmap lock. Reported and tested by: pho Reviewed by: alc Sponsored by: The FreeBSD Foundation Modified: head/sys/amd64/amd64/pmap.c head/sys/amd64/include/pmap.h Modified: head/sys/amd64/amd64/pmap.c ============================================================================== --- head/sys/amd64/amd64/pmap.c Wed Aug 7 16:01:45 2013 (r254063) +++ head/sys/amd64/amd64/pmap.c Wed Aug 7 16:33:15 2013 (r254064) @@ -2183,6 +2183,7 @@ reclaim_pv_chunk(pmap_t locked_pmap, str vm_page_aflag_set(m, PGA_REFERENCED); CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); @@ -2470,6 +2471,7 @@ pmap_pvh_remove(struct md_page *pvh, pma TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { if (pmap == PV_PMAP(pv) && va == pv->pv_va) { TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); + pvh->pv_gen++; break; } } @@ -2509,6 +2511,7 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offse KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found")); m = PHYS_TO_VM_PAGE(pa); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; /* Instantiate the remaining NPTEPG - 1 pv entries. */ PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1)); va_last = va + NBPDR - PAGE_SIZE; @@ -2527,6 +2530,7 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offse KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_pv_demote_pde: page %p is not managed", m)); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; if (va == va_last) goto out; } @@ -2575,6 +2579,7 @@ pmap_pv_promote_pde(pmap_t pmap, vm_offs KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found")); pvh = pa_to_pvh(pa); TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); + pvh->pv_gen++; /* Free the remaining NPTEPG - 1 pv entries. */ va_last = va + NBPDR - PAGE_SIZE; do { @@ -2616,6 +2621,7 @@ pmap_try_insert_pv_entry(pmap_t pmap, vm pv->pv_va = va; CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; return (TRUE); } else return (FALSE); @@ -2640,6 +2646,7 @@ pmap_pv_insert_pde(pmap_t pmap, vm_offse CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); pvh = pa_to_pvh(pa); TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); + pvh->pv_gen++; return (TRUE); } else return (FALSE); @@ -3156,6 +3163,7 @@ small_mappings: pmap_unuse_pt(pmap, pv->pv_va, *pde, &free); pmap_invalidate_page(pmap, pv->pv_va); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; free_pv_entry(pmap, pv); PMAP_UNLOCK(pmap); } @@ -3601,6 +3609,7 @@ retry: pv->pv_va = va; CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; if ((newpte & PG_RW) != 0) vm_page_aflag_set(m, PGA_WRITEABLE); } @@ -4536,6 +4545,7 @@ pmap_remove_pages(pmap_t pmap) pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE); pvh = pa_to_pvh(tpte & PG_PS_FRAME); TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); + pvh->pv_gen++; if (TAILQ_EMPTY(&pvh->pv_list)) { for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++) if ((mt->aflags & PGA_WRITEABLE) != 0 && @@ -4555,6 +4565,7 @@ pmap_remove_pages(pmap_t pmap) } else { pmap_resident_count_dec(pmap, 1); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; if ((m->aflags & PGA_WRITEABLE) != 0 && TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { @@ -4793,25 +4804,40 @@ pmap_ts_referenced(vm_page_t m) struct md_page *pvh; pv_entry_t pv, pvf, pvn; pmap_t pmap; + struct rwlock *lock; pd_entry_t oldpde, *pde; pt_entry_t *pte; vm_offset_t va; - int rtval = 0; + int rtval, pvh_gen, md_gen; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_ts_referenced: page %p is not managed", m)); - rw_wlock(&pvh_global_lock); + rw_rlock(&pvh_global_lock); + lock = VM_PAGE_TO_PV_LIST_LOCK(m); + pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); + rtval = 0; +retry: + rw_wlock(lock); if ((m->flags & PG_FICTITIOUS) != 0) goto small_mappings; - pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, pvn) { pmap = PV_PMAP(pv); - PMAP_LOCK(pmap); + if (!PMAP_TRYLOCK(pmap)) { + pvh_gen = pvh->pv_gen; + rw_wunlock(lock); + PMAP_LOCK(pmap); + rw_wlock(lock); + if (pvh_gen != pvh->pv_gen) { + PMAP_UNLOCK(pmap); + rw_wunlock(lock); + goto retry; + } + } va = pv->pv_va; pde = pmap_pde(pmap, va); oldpde = *pde; if ((oldpde & PG_A) != 0) { - if (pmap_demote_pde(pmap, pde, va)) { + if (pmap_demote_pde_locked(pmap, pde, va, &lock)) { if ((oldpde & PG_W) == 0) { /* * Remove the mapping to a single page @@ -4823,7 +4849,10 @@ pmap_ts_referenced(vm_page_t m) */ va += VM_PAGE_TO_PHYS(m) - (oldpde & PG_PS_FRAME); - pmap_remove_page(pmap, va, pde, NULL); + pte = pmap_pde_to_pte(pde, va); + pmap_remove_pte(pmap, pte, va, *pde, + NULL, &lock); + pmap_invalidate_page(pmap, va); rtval++; if (rtval > 4) { PMAP_UNLOCK(pmap); @@ -4831,6 +4860,9 @@ pmap_ts_referenced(vm_page_t m) } } } + KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), + ("inconsistent pv lock %p %p for page %p", + lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); } PMAP_UNLOCK(pmap); } @@ -4841,8 +4873,21 @@ small_mappings: pvn = TAILQ_NEXT(pv, pv_next); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; pmap = PV_PMAP(pv); - PMAP_LOCK(pmap); + if (!PMAP_TRYLOCK(pmap)) { + pvh_gen = pvh->pv_gen; + md_gen = m->md.pv_gen; + rw_wunlock(lock); + PMAP_LOCK(pmap); + rw_wlock(lock); + if (pvh_gen != pvh->pv_gen || + md_gen != m->md.pv_gen) { + PMAP_UNLOCK(pmap); + rw_wunlock(lock); + goto retry; + } + } pde = pmap_pde(pmap, pv->pv_va); KASSERT((*pde & PG_PS) == 0, ("pmap_ts_referenced:" " found a 2mpage in page %p's pv list", m)); @@ -4858,7 +4903,8 @@ small_mappings: } while ((pv = pvn) != NULL && pv != pvf); } out: - rw_wunlock(&pvh_global_lock); + rw_wunlock(lock); + rw_runlock(&pvh_global_lock); return (rtval); } Modified: head/sys/amd64/include/pmap.h ============================================================================== --- head/sys/amd64/include/pmap.h Wed Aug 7 16:01:45 2013 (r254063) +++ head/sys/amd64/include/pmap.h Wed Aug 7 16:33:15 2013 (r254064) @@ -233,6 +233,7 @@ struct pv_chunk; struct md_page { TAILQ_HEAD(,pv_entry) pv_list; + int pv_gen; int pat_mode; };