Date: Thu, 22 Aug 2013 01:54:53 +0000 (UTC) From: Neel Natu <neel@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r254633 - in projects/bhyve_npt_pmap/sys/amd64: amd64 include Message-ID: <201308220154.r7M1srM9050130@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: neel Date: Thu Aug 22 01:54:52 2013 New Revision: 254633 URL: http://svnweb.freebsd.org/changeset/base/254633 Log: IFC @254064 Modified: projects/bhyve_npt_pmap/sys/amd64/amd64/pmap.c projects/bhyve_npt_pmap/sys/amd64/include/pmap.h Directory Properties: projects/bhyve_npt_pmap/ (props changed) projects/bhyve_npt_pmap/sys/ (props changed) Modified: projects/bhyve_npt_pmap/sys/amd64/amd64/pmap.c ============================================================================== --- projects/bhyve_npt_pmap/sys/amd64/amd64/pmap.c Thu Aug 22 00:51:56 2013 (r254632) +++ projects/bhyve_npt_pmap/sys/amd64/amd64/pmap.c Thu Aug 22 01:54:52 2013 (r254633) @@ -2547,6 +2547,7 @@ reclaim_pv_chunk(pmap_t locked_pmap, str vm_page_aflag_set(m, PGA_REFERENCED); CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); @@ -2834,6 +2835,7 @@ pmap_pvh_remove(struct md_page *pvh, pma TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { if (pmap == PV_PMAP(pv) && va == pv->pv_va) { TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); + pvh->pv_gen++; break; } } @@ -2873,6 +2875,7 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offse KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found")); m = PHYS_TO_VM_PAGE(pa); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; /* Instantiate the remaining NPTEPG - 1 pv entries. */ PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1)); va_last = va + NBPDR - PAGE_SIZE; @@ -2891,6 +2894,7 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offse KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_pv_demote_pde: page %p is not managed", m)); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; if (va == va_last) goto out; } @@ -2939,6 +2943,7 @@ pmap_pv_promote_pde(pmap_t pmap, vm_offs KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found")); pvh = pa_to_pvh(pa); TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); + pvh->pv_gen++; /* Free the remaining NPTEPG - 1 pv entries. */ va_last = va + NBPDR - PAGE_SIZE; do { @@ -2980,6 +2985,7 @@ pmap_try_insert_pv_entry(pmap_t pmap, vm pv->pv_va = va; CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; return (TRUE); } else return (FALSE); @@ -3004,6 +3010,7 @@ pmap_pv_insert_pde(pmap_t pmap, vm_offse CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); pvh = pa_to_pvh(pa); TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); + pvh->pv_gen++; return (TRUE); } else return (FALSE); @@ -3549,6 +3556,7 @@ small_mappings: pmap_unuse_pt(pmap, pv->pv_va, *pde, &free); pmap_invalidate_page(pmap, pv->pv_va); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; free_pv_entry(pmap, pv); PMAP_UNLOCK(pmap); } @@ -4066,6 +4074,7 @@ retry: pv->pv_va = va; CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; if (pmap_writeable_mapping(pmap, newpte)) vm_page_aflag_set(m, PGA_WRITEABLE); } @@ -5048,6 +5057,7 @@ pmap_remove_pages(pmap_t pmap) pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE); pvh = pa_to_pvh(tpte & PG_PS_FRAME); TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); + pvh->pv_gen++; if (TAILQ_EMPTY(&pvh->pv_list)) { for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++) if ((mt->aflags & PGA_WRITEABLE) != 0 && @@ -5067,6 +5077,7 @@ pmap_remove_pages(pmap_t pmap) } else { pmap_resident_count_dec(pmap, 1); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; if ((m->aflags & PGA_WRITEABLE) != 0 && TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { @@ -5313,27 +5324,42 @@ pmap_ts_referenced(vm_page_t m) struct md_page *pvh; pv_entry_t pv, pvf, pvn; pmap_t pmap; + struct rwlock *lock; pd_entry_t oldpde, *pde; pt_entry_t *pte, PG_A; vm_offset_t va; - int rtval = 0; + int rtval, pvh_gen, md_gen; vm_page_t free = NULL; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_ts_referenced: page %p is not managed", m)); - rw_wlock(&pvh_global_lock); + rw_rlock(&pvh_global_lock); + lock = VM_PAGE_TO_PV_LIST_LOCK(m); + pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); + rtval = 0; +retry: + rw_wlock(lock); if ((m->flags & PG_FICTITIOUS) != 0) goto small_mappings; - pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, pvn) { pmap = PV_PMAP(pv); - PMAP_LOCK(pmap); + if (!PMAP_TRYLOCK(pmap)) { + pvh_gen = pvh->pv_gen; + rw_wunlock(lock); + PMAP_LOCK(pmap); + rw_wlock(lock); + if (pvh_gen != pvh->pv_gen) { + PMAP_UNLOCK(pmap); + rw_wunlock(lock); + goto retry; + } + } PG_A = pmap_accessed_bit(pmap); va = pv->pv_va; pde = pmap_pde(pmap, va); oldpde = *pde; if ((oldpde & PG_A) != 0) { - if (pmap_demote_pde(pmap, pde, va)) { + if (pmap_demote_pde_locked(pmap, pde, va, &lock)) { if ((oldpde & PG_W) == 0) { /* * Remove the mapping to a single page @@ -5345,7 +5371,10 @@ pmap_ts_referenced(vm_page_t m) */ va += VM_PAGE_TO_PHYS(m) - (oldpde & PG_PS_FRAME); - pmap_remove_page(pmap, va, pde, NULL); + pte = pmap_pde_to_pte(pde, va); + pmap_remove_pte(pmap, pte, va, *pde, + NULL, &lock); + pmap_invalidate_page(pmap, va); rtval++; if (rtval > 4) { PMAP_UNLOCK(pmap); @@ -5353,6 +5382,9 @@ pmap_ts_referenced(vm_page_t m) } } } + KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), + ("inconsistent pv lock %p %p for page %p", + lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); } PMAP_UNLOCK(pmap); } @@ -5365,10 +5397,24 @@ small_mappings: pvn = TAILQ_NEXT(pv, pv_next); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; pmap = PV_PMAP(pv); - PMAP_LOCK(pmap); + if (!PMAP_TRYLOCK(pmap)) { + pvh_gen = pvh->pv_gen; + md_gen = m->md.pv_gen; + rw_wunlock(lock); + PMAP_LOCK(pmap); + rw_wlock(lock); + if (pvh_gen != pvh->pv_gen || + md_gen != m->md.pv_gen) { + PMAP_UNLOCK(pmap); + rw_wunlock(lock); + goto retry; + } + } PG_A = pmap_accessed_bit(pmap); - pde = pmap_pde(pmap, pv->pv_va); + va = pv->pv_va; + pde = pmap_pde(pmap, va); KASSERT((*pde & PG_PS) == 0, ("pmap_ts_referenced:" " found a 2mpage in page %p's pv list", m)); pte = pmap_pde_to_pte(pde, pv->pv_va); @@ -5381,8 +5427,10 @@ small_mappings: * hard work for unwired pages only. */ if ((*pte & PG_W) == 0) { - pmap_remove_page(pmap, - pv->pv_va, pde, &free); + pte = pmap_pde_to_pte(pde, va); + pmap_remove_pte(pmap, pte, va, + *pde, &free, &lock); + pmap_invalidate_page(pmap, va); if (pvf == pv) pvf = NULL; } @@ -5393,12 +5441,17 @@ small_mappings: rtval++; if (rtval > 4) pvn = NULL; + + KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), + ("inconsistent pv lock %p %p for page %p", + lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); } PMAP_UNLOCK(pmap); } while ((pv = pvn) != NULL && pv != pvf); } out: - rw_wunlock(&pvh_global_lock); + rw_wunlock(lock); + rw_runlock(&pvh_global_lock); pmap_free_zero_pages(free); return (rtval); } Modified: projects/bhyve_npt_pmap/sys/amd64/include/pmap.h ============================================================================== --- projects/bhyve_npt_pmap/sys/amd64/include/pmap.h Thu Aug 22 00:51:56 2013 (r254632) +++ projects/bhyve_npt_pmap/sys/amd64/include/pmap.h Thu Aug 22 01:54:52 2013 (r254633) @@ -243,6 +243,7 @@ struct pv_chunk; struct md_page { TAILQ_HEAD(,pv_entry) pv_list; + int pv_gen; int pat_mode; };
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201308220154.r7M1srM9050130>