Skip site navigation (1)Skip section navigation (2)
Date:      Wed, 7 Aug 2013 16:33:15 +0000 (UTC)
From:      Konstantin Belousov <kib@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r254064 - in head/sys/amd64: amd64 include
Message-ID:  <201308071633.r77GXF0T092188@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: kib
Date: Wed Aug  7 16:33:15 2013
New Revision: 254064
URL: http://svnweb.freebsd.org/changeset/base/254064

Log:
  Change the pmap_ts_referenced() method of amd64 pmap to use shared
  pvh_global_lock.  This allows the method to be executed in parallel,
  avoiding undue contention on the pvh_global_lock for the multithreaded
  pagedaemon.
  
  The pmap_ts_referenced() function has to inspect the page mappings for
  several pmaps, which need to be locked while pv list lock is owned.
  This contradicts to the lock order, where pmap lock is before pv list
  lock.  Introduce the generation count for the pv list of the page or
  superpage, which indicate any change in the pv list, and, as usual,
  perform restart of the iteration if generation changed while pv lock
  was dropped for blocking acquire of a pmap lock.
  
  Reported and tested by:	pho
  Reviewed by:	alc
  Sponsored by:	The FreeBSD Foundation

Modified:
  head/sys/amd64/amd64/pmap.c
  head/sys/amd64/include/pmap.h

Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c	Wed Aug  7 16:01:45 2013	(r254063)
+++ head/sys/amd64/amd64/pmap.c	Wed Aug  7 16:33:15 2013	(r254064)
@@ -2183,6 +2183,7 @@ reclaim_pv_chunk(pmap_t locked_pmap, str
 					vm_page_aflag_set(m, PGA_REFERENCED);
 				CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
 				TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
+				m->md.pv_gen++;
 				if (TAILQ_EMPTY(&m->md.pv_list) &&
 				    (m->flags & PG_FICTITIOUS) == 0) {
 					pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
@@ -2470,6 +2471,7 @@ pmap_pvh_remove(struct md_page *pvh, pma
 	TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 		if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
 			TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
+			pvh->pv_gen++;
 			break;
 		}
 	}
@@ -2509,6 +2511,7 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offse
 	KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
 	m = PHYS_TO_VM_PAGE(pa);
 	TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
+	m->md.pv_gen++;
 	/* Instantiate the remaining NPTEPG - 1 pv entries. */
 	PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1));
 	va_last = va + NBPDR - PAGE_SIZE;
@@ -2527,6 +2530,7 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offse
 				KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 			    ("pmap_pv_demote_pde: page %p is not managed", m));
 				TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
+				m->md.pv_gen++;
 				if (va == va_last)
 					goto out;
 			}
@@ -2575,6 +2579,7 @@ pmap_pv_promote_pde(pmap_t pmap, vm_offs
 	KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
 	pvh = pa_to_pvh(pa);
 	TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
+	pvh->pv_gen++;
 	/* Free the remaining NPTEPG - 1 pv entries. */
 	va_last = va + NBPDR - PAGE_SIZE;
 	do {
@@ -2616,6 +2621,7 @@ pmap_try_insert_pv_entry(pmap_t pmap, vm
 		pv->pv_va = va;
 		CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
 		TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
+		m->md.pv_gen++;
 		return (TRUE);
 	} else
 		return (FALSE);
@@ -2640,6 +2646,7 @@ pmap_pv_insert_pde(pmap_t pmap, vm_offse
 		CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
 		pvh = pa_to_pvh(pa);
 		TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
+		pvh->pv_gen++;
 		return (TRUE);
 	} else
 		return (FALSE);
@@ -3156,6 +3163,7 @@ small_mappings:
 		pmap_unuse_pt(pmap, pv->pv_va, *pde, &free);
 		pmap_invalidate_page(pmap, pv->pv_va);
 		TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
+		m->md.pv_gen++;
 		free_pv_entry(pmap, pv);
 		PMAP_UNLOCK(pmap);
 	}
@@ -3601,6 +3609,7 @@ retry:
 		pv->pv_va = va;
 		CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa);
 		TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
+		m->md.pv_gen++;
 		if ((newpte & PG_RW) != 0)
 			vm_page_aflag_set(m, PGA_WRITEABLE);
 	}
@@ -4536,6 +4545,7 @@ pmap_remove_pages(pmap_t pmap)
 					pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE);
 					pvh = pa_to_pvh(tpte & PG_PS_FRAME);
 					TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
+					pvh->pv_gen++;
 					if (TAILQ_EMPTY(&pvh->pv_list)) {
 						for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 							if ((mt->aflags & PGA_WRITEABLE) != 0 &&
@@ -4555,6 +4565,7 @@ pmap_remove_pages(pmap_t pmap)
 				} else {
 					pmap_resident_count_dec(pmap, 1);
 					TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
+					m->md.pv_gen++;
 					if ((m->aflags & PGA_WRITEABLE) != 0 &&
 					    TAILQ_EMPTY(&m->md.pv_list) &&
 					    (m->flags & PG_FICTITIOUS) == 0) {
@@ -4793,25 +4804,40 @@ pmap_ts_referenced(vm_page_t m)
 	struct md_page *pvh;
 	pv_entry_t pv, pvf, pvn;
 	pmap_t pmap;
+	struct rwlock *lock;
 	pd_entry_t oldpde, *pde;
 	pt_entry_t *pte;
 	vm_offset_t va;
-	int rtval = 0;
+	int rtval, pvh_gen, md_gen;
 
 	KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 	    ("pmap_ts_referenced: page %p is not managed", m));
-	rw_wlock(&pvh_global_lock);
+	rw_rlock(&pvh_global_lock);
+	lock = VM_PAGE_TO_PV_LIST_LOCK(m);
+	pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
+	rtval = 0;
+retry:
+	rw_wlock(lock);
 	if ((m->flags & PG_FICTITIOUS) != 0)
 		goto small_mappings;
-	pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 	TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, pvn) {
 		pmap = PV_PMAP(pv);
-		PMAP_LOCK(pmap);
+		if (!PMAP_TRYLOCK(pmap)) {
+			pvh_gen = pvh->pv_gen;
+			rw_wunlock(lock);
+			PMAP_LOCK(pmap);
+			rw_wlock(lock);
+			if (pvh_gen != pvh->pv_gen) {
+				PMAP_UNLOCK(pmap);
+				rw_wunlock(lock);
+				goto retry;
+			}
+		}
 		va = pv->pv_va;
 		pde = pmap_pde(pmap, va);
 		oldpde = *pde;
 		if ((oldpde & PG_A) != 0) {
-			if (pmap_demote_pde(pmap, pde, va)) {
+			if (pmap_demote_pde_locked(pmap, pde, va, &lock)) {
 				if ((oldpde & PG_W) == 0) {
 					/*
 					 * Remove the mapping to a single page
@@ -4823,7 +4849,10 @@ pmap_ts_referenced(vm_page_t m)
 					 */
 					va += VM_PAGE_TO_PHYS(m) - (oldpde &
 					    PG_PS_FRAME);
-					pmap_remove_page(pmap, va, pde, NULL);
+					pte = pmap_pde_to_pte(pde, va);
+					pmap_remove_pte(pmap, pte, va, *pde,
+					    NULL, &lock);
+					pmap_invalidate_page(pmap, va);
 					rtval++;
 					if (rtval > 4) {
 						PMAP_UNLOCK(pmap);
@@ -4831,6 +4860,9 @@ pmap_ts_referenced(vm_page_t m)
 					}
 				}
 			}
+			KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m),
+			    ("inconsistent pv lock %p %p for page %p",
+			    lock, VM_PAGE_TO_PV_LIST_LOCK(m), m));
 		}
 		PMAP_UNLOCK(pmap);
 	}
@@ -4841,8 +4873,21 @@ small_mappings:
 			pvn = TAILQ_NEXT(pv, pv_next);
 			TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 			TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
+			m->md.pv_gen++;
 			pmap = PV_PMAP(pv);
-			PMAP_LOCK(pmap);
+			if (!PMAP_TRYLOCK(pmap)) {
+				pvh_gen = pvh->pv_gen;
+				md_gen = m->md.pv_gen;
+				rw_wunlock(lock);
+				PMAP_LOCK(pmap);
+				rw_wlock(lock);
+				if (pvh_gen != pvh->pv_gen ||
+				    md_gen != m->md.pv_gen) {
+					PMAP_UNLOCK(pmap);
+					rw_wunlock(lock);
+					goto retry;
+				}
+			}
 			pde = pmap_pde(pmap, pv->pv_va);
 			KASSERT((*pde & PG_PS) == 0, ("pmap_ts_referenced:"
 			    " found a 2mpage in page %p's pv list", m));
@@ -4858,7 +4903,8 @@ small_mappings:
 		} while ((pv = pvn) != NULL && pv != pvf);
 	}
 out:
-	rw_wunlock(&pvh_global_lock);
+	rw_wunlock(lock);
+	rw_runlock(&pvh_global_lock);
 	return (rtval);
 }
 

Modified: head/sys/amd64/include/pmap.h
==============================================================================
--- head/sys/amd64/include/pmap.h	Wed Aug  7 16:01:45 2013	(r254063)
+++ head/sys/amd64/include/pmap.h	Wed Aug  7 16:33:15 2013	(r254064)
@@ -233,6 +233,7 @@ struct	pv_chunk;
 
 struct md_page {
 	TAILQ_HEAD(,pv_entry)	pv_list;
+	int			pv_gen;
 	int			pat_mode;
 };
 



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201308071633.r77GXF0T092188>