From owner-svn-src-user@FreeBSD.ORG Tue Feb 17 03:58:16 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 7320A106567A; Tue, 17 Feb 2009 03:58:16 +0000 (UTC) (envelope-from alc@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 6013E8FC23; Tue, 17 Feb 2009 03:58:16 +0000 (UTC) (envelope-from alc@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n1H3wG67002800; Tue, 17 Feb 2009 03:58:16 GMT (envelope-from alc@svn.freebsd.org) Received: (from alc@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n1H3wFS5002787; Tue, 17 Feb 2009 03:58:15 GMT (envelope-from alc@svn.freebsd.org) Message-Id: <200902170358.n1H3wFS5002787@svn.freebsd.org> From: Alan Cox Date: Tue, 17 Feb 2009 03:58:15 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r188703 - in user/alc/pagelock/sys: amd64/amd64 ddb kern security/mac vm X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 17 Feb 2009 03:58:17 -0000 Author: alc Date: Tue Feb 17 03:58:15 2009 New Revision: 188703 URL: http://svn.freebsd.org/changeset/base/188703 Log: Experimental pmap locking changes Modified: user/alc/pagelock/sys/amd64/amd64/pmap.c user/alc/pagelock/sys/ddb/db_watch.c user/alc/pagelock/sys/kern/kern_subr.c user/alc/pagelock/sys/kern/uipc_shm.c user/alc/pagelock/sys/kern/vfs_bio.c user/alc/pagelock/sys/security/mac/mac_process.c user/alc/pagelock/sys/vm/vm_fault.c user/alc/pagelock/sys/vm/vm_map.c user/alc/pagelock/sys/vm/vm_object.c user/alc/pagelock/sys/vm/vm_object.h user/alc/pagelock/sys/vm/vm_page.c user/alc/pagelock/sys/vm/vm_pageout.c user/alc/pagelock/sys/vm/vnode_pager.c Modified: user/alc/pagelock/sys/amd64/amd64/pmap.c ============================================================================== --- user/alc/pagelock/sys/amd64/amd64/pmap.c Tue Feb 17 03:06:40 2009 (r188702) +++ user/alc/pagelock/sys/amd64/amd64/pmap.c Tue Feb 17 03:58:15 2009 (r188703) @@ -252,10 +252,13 @@ static void pmap_insert_entry(pmap_t pma static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m); -static vm_page_t pmap_allocpde(pmap_t pmap, vm_offset_t va, int flags); -static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags); +static vm_page_t pmap_allocpde(pmap_t pmap, vm_offset_t va, vm_object_t object, + int flags); +static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, vm_object_t object, + int flags); -static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags); +static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, + vm_object_t object, int flags); static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t* free); static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, vm_page_t *); @@ -1452,7 +1455,8 @@ pmap_pinit(pmap_t pmap) * race conditions. */ static vm_page_t -_pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags) +_pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, vm_object_t object, + int flags) { vm_page_t m, pdppg, pdpg; @@ -1468,7 +1472,9 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t if (flags & M_WAITOK) { PMAP_UNLOCK(pmap); vm_page_unlock_queues(); + vm_object_unlock_all(object); VM_WAIT; + vm_object_lock_all(object); vm_page_lock_queues(); PMAP_LOCK(pmap); } @@ -1512,7 +1518,7 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t if ((*pml4 & PG_V) == 0) { /* Have to allocate a new pdp, recurse */ if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml4index, - flags) == NULL) { + object, flags) == NULL) { --m->wire_count; vm_page_free(m); return (NULL); @@ -1544,7 +1550,7 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t if ((*pml4 & PG_V) == 0) { /* Have to allocate a new pd, recurse */ if (_pmap_allocpte(pmap, NUPDE + pdpindex, - flags) == NULL) { + object, flags) == NULL) { --m->wire_count; vm_page_free(m); return (NULL); @@ -1557,7 +1563,7 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t if ((*pdp & PG_V) == 0) { /* Have to allocate a new pd, recurse */ if (_pmap_allocpte(pmap, NUPDE + pdpindex, - flags) == NULL) { + object, flags) == NULL) { --m->wire_count; vm_page_free(m); return (NULL); @@ -1579,7 +1585,7 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t } static vm_page_t -pmap_allocpde(pmap_t pmap, vm_offset_t va, int flags) +pmap_allocpde(pmap_t pmap, vm_offset_t va, vm_object_t object, int flags) { vm_pindex_t pdpindex, ptepindex; pdp_entry_t *pdpe; @@ -1598,7 +1604,7 @@ retry: /* Allocate a pd page. */ ptepindex = pmap_pde_pindex(va); pdpindex = ptepindex >> NPDPEPGSHIFT; - pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, flags); + pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, object, flags); if (pdpg == NULL && (flags & M_WAITOK)) goto retry; } @@ -1606,7 +1612,7 @@ retry: } static vm_page_t -pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags) +pmap_allocpte(pmap_t pmap, vm_offset_t va, vm_object_t object, int flags) { vm_pindex_t ptepindex; pd_entry_t *pd; @@ -1652,7 +1658,7 @@ retry: * Here if the pte page isn't mapped, or if it has been * deallocated. */ - m = _pmap_allocpte(pmap, ptepindex, flags); + m = _pmap_allocpte(pmap, ptepindex, object, flags); if (m == NULL && (flags & M_WAITOK)) goto retry; } @@ -1869,6 +1875,7 @@ pmap_collect(pmap_t locked_pmap, struct vm_offset_t va; vm_page_t m, free; + mtx_assert(&vm_page_queue_mtx, MA_OWNED); /* XXX */ TAILQ_FOREACH(m, &vpq->pl, pageq) { if (m->hold_count || m->busy) continue; @@ -1920,11 +1927,10 @@ free_pv_entry(pmap_t pmap, pv_entry_t pv struct pv_chunk *pc; int idx, field, bit; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); PV_STAT(pv_entry_frees++); PV_STAT(pv_entry_spare++); - pv_entry_count--; + atomic_subtract_int(&pv_entry_count, 1); pc = pv_to_chunk(pv); idx = pv - &pc->pc_pventry[0]; field = idx / 64; @@ -1964,9 +1970,8 @@ get_pv_entry(pmap_t pmap, int try) vm_page_t m; PMAP_LOCK_ASSERT(pmap, MA_OWNED); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); PV_STAT(pv_entry_allocs++); - pv_entry_count++; + atomic_add_int(&pv_entry_count, 1); if (pv_entry_count > pv_entry_high_water) if (ratecheck(&lastprint, &printinterval)) printf("Approaching the limit on PV entries, consider " @@ -2001,7 +2006,7 @@ retry: VM_ALLOC_WIRED); if (m == NULL) { if (try) { - pv_entry_count--; + atomic_subtract_int(&pv_entry_count, 1); PV_STAT(pc_chunk_tryfail++); return (NULL); } @@ -2047,7 +2052,6 @@ pmap_pvh_remove(struct md_page *pvh, pma { pv_entry_t pv; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) { if (pmap == PV_PMAP(pv) && va == pv->pv_va) { TAILQ_REMOVE(&pvh->pv_list, pv, pv_list); @@ -2070,7 +2074,6 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offse vm_offset_t va_last; vm_page_t m; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); KASSERT((pa & PDRMASK) == 0, ("pmap_pv_demote_pde: pa is not 2mpage aligned")); @@ -2083,6 +2086,7 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offse pv = pmap_pvh_remove(pvh, pmap, va); KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found")); m = PHYS_TO_VM_PAGE(pa); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); /* Instantiate the remaining NPTEPG - 1 pv entries. */ va_last = va + NBPDR - PAGE_SIZE; @@ -2108,7 +2112,6 @@ pmap_pv_promote_pde(pmap_t pmap, vm_offs vm_offset_t va_last; vm_page_t m; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); KASSERT((pa & PDRMASK) == 0, ("pmap_pv_promote_pde: pa is not 2mpage aligned")); @@ -2121,6 +2124,7 @@ pmap_pv_promote_pde(pmap_t pmap, vm_offs */ m = PHYS_TO_VM_PAGE(pa); va = trunc_2mpage(va); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); pv = pmap_pvh_remove(&m->md, pmap, va); KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found")); pvh = pa_to_pvh(pa); @@ -2130,6 +2134,7 @@ pmap_pv_promote_pde(pmap_t pmap, vm_offs do { m++; va += PAGE_SIZE; + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); pmap_pvh_free(&m->md, pmap, va); } while (va < va_last); } @@ -2154,7 +2159,7 @@ pmap_remove_entry(pmap_t pmap, vm_page_t { struct md_page *pvh; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); pmap_pvh_free(&m->md, pmap, va); if (TAILQ_EMPTY(&m->md.pv_list)) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); @@ -2172,8 +2177,7 @@ pmap_insert_entry(pmap_t pmap, vm_offset { pv_entry_t pv; - PMAP_LOCK_ASSERT(pmap, MA_OWNED); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); pv = get_pv_entry(pmap, FALSE); pv->pv_va = va; TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); @@ -2187,8 +2191,7 @@ pmap_try_insert_pv_entry(pmap_t pmap, vm { pv_entry_t pv; - PMAP_LOCK_ASSERT(pmap, MA_OWNED); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); if (pv_entry_count < pv_entry_high_water && (pv = get_pv_entry(pmap, TRUE)) != NULL) { pv->pv_va = va; @@ -2207,7 +2210,6 @@ pmap_pv_insert_pde(pmap_t pmap, vm_offse struct md_page *pvh; pv_entry_t pv; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); if (pv_entry_count < pv_entry_high_water && (pv = get_pv_entry(pmap, TRUE)) != NULL) { pv->pv_va = va; @@ -2601,7 +2603,7 @@ pmap_remove_all(vm_page_t m) KASSERT((m->flags & PG_FICTITIOUS) == 0, ("pmap_remove_all: page %p is fictitious", m)); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) { va = pv->pv_va; @@ -2969,7 +2971,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va, * resident, we are creating it here. */ if (va < VM_MAXUSER_ADDRESS) { - mpte = pmap_allocpte(pmap, va, M_WAITOK); + mpte = pmap_allocpte(pmap, va, m->object, M_WAITOK); } pde = pmap_pde(pmap, va); @@ -3124,9 +3126,9 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t pd_entry_t *pde, newpde; vm_page_t free, mpde; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); - if ((mpde = pmap_allocpde(pmap, va, M_NOWAIT)) == NULL) { + if ((mpde = pmap_allocpde(pmap, va, NULL, M_NOWAIT)) == NULL) { CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" " in pmap %p", va, pmap); return (FALSE); @@ -3249,7 +3251,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_ KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0, ("pmap_enter_quick_locked: managed mapping within the clean submap")); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* @@ -3283,7 +3285,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_ mpte->wire_count++; } else { mpte = _pmap_allocpte(pmap, ptepindex, - M_NOWAIT); + NULL, M_NOWAIT); if (mpte == NULL) return (mpte); } @@ -3412,7 +3414,7 @@ retry: PMAP_LOCK(pmap); for (va = addr; va < addr + size; va += NBPDR) { while ((pdpg = - pmap_allocpde(pmap, va, M_NOWAIT)) == NULL) { + pmap_allocpde(pmap, va, NULL, M_NOWAIT)) == NULL) { PMAP_UNLOCK(pmap); vm_page_busy(p); VM_OBJECT_UNLOCK(object); @@ -3519,7 +3521,6 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pm if (!pmap_is_current(src_pmap)) return; - vm_page_lock_queues(); if (dst_pmap < src_pmap) { PMAP_LOCK(dst_pmap); PMAP_LOCK(src_pmap); @@ -3563,7 +3564,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pm continue; if (srcptepaddr & PG_PS) { - dstmpde = pmap_allocpde(dst_pmap, addr, M_NOWAIT); + dstmpde = pmap_allocpde(dst_pmap, addr, NULL, M_NOWAIT); if (dstmpde == NULL) break; pde = (pd_entry_t *) @@ -3595,7 +3596,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pm * we only virtual copy managed pages */ if ((ptetemp & PG_MANAGED) != 0) { - dstmpte = pmap_allocpte(dst_pmap, addr, + dstmpte = pmap_allocpte(dst_pmap, addr, NULL, M_NOWAIT); if (dstmpte == NULL) break; @@ -3629,7 +3630,6 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pm src_pte++; } } - vm_page_unlock_queues(); PMAP_UNLOCK(src_pmap); PMAP_UNLOCK(dst_pmap); } @@ -3709,7 +3709,7 @@ pmap_page_exists_quick(pmap_t pmap, vm_p if (m->flags & PG_FICTITIOUS) return FALSE; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (PV_PMAP(pv) == pmap) { return TRUE; @@ -3745,6 +3745,7 @@ pmap_page_wired_mappings(vm_page_t m) count = 0; if ((m->flags & PG_FICTITIOUS) != 0) return (count); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); count = pmap_pvh_wired_mappings(&m->md, count); return (pmap_pvh_wired_mappings(pa_to_pvh(VM_PAGE_TO_PHYS(m)), count)); } @@ -3761,7 +3762,6 @@ pmap_pvh_wired_mappings(struct md_page * pt_entry_t *pte; pv_entry_t pv; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); @@ -3784,7 +3784,7 @@ pmap_page_is_mapped(vm_page_t m) if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return (FALSE); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); if (TAILQ_EMPTY(&m->md.pv_list)) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); return (!TAILQ_EMPTY(&pvh->pv_list)); @@ -3805,6 +3805,7 @@ pmap_remove_pages(pmap_t pmap) { pd_entry_t *pde; pt_entry_t *pte, tpte; + vm_object_t locked_object; vm_page_t free = NULL; vm_page_t m, mpte, mt; pv_entry_t pv; @@ -3819,6 +3820,8 @@ pmap_remove_pages(pmap_t pmap) printf("warning: pmap_remove_pages called with non-current pmap\n"); return; } + locked_object = NULL; +restart: vm_page_lock_queues(); PMAP_LOCK(pmap); TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { @@ -3865,6 +3868,18 @@ pmap_remove_pages(pmap_t pmap) KASSERT(m < &vm_page_array[vm_page_array_size], ("pmap_remove_pages: bad tpte %#jx", (uintmax_t)tpte)); + if (m->object != locked_object) { + if (locked_object != NULL) + VM_OBJECT_UNLOCK(locked_object); + locked_object = m->object; + if (!VM_OBJECT_TRYLOCK(locked_object)) { + pmap_invalidate_all(pmap); + vm_page_unlock_queues(); + PMAP_UNLOCK(pmap); + VM_OBJECT_LOCK(locked_object); + goto restart; + } + } pte_clear(pte); @@ -3882,7 +3897,7 @@ pmap_remove_pages(pmap_t pmap) /* Mark free */ PV_STAT(pv_entry_frees++); PV_STAT(pv_entry_spare++); - pv_entry_count--; + atomic_subtract_int(&pv_entry_count, 1); pc->pc_map[field] |= bitmask; if ((tpte & PG_PS) != 0) { pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; @@ -3916,6 +3931,10 @@ pmap_remove_pages(pmap_t pmap) } } } + if (locked_object != NULL) { + VM_OBJECT_UNLOCK(locked_object); + locked_object = NULL; + } if (allfree) { PV_STAT(pv_entry_spare -= _NPCPV); PV_STAT(pc_chunk_count--); @@ -3945,6 +3964,7 @@ pmap_is_modified(vm_page_t m) if (m->flags & PG_FICTITIOUS) return (FALSE); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); if (pmap_is_modified_pvh(&m->md)) return (TRUE); return (pmap_is_modified_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m)))); @@ -3963,7 +3983,6 @@ pmap_is_modified_pvh(struct md_page *pvh pmap_t pmap; boolean_t rv; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); rv = FALSE; TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) { pmap = PV_PMAP(pv); @@ -4017,7 +4036,7 @@ pmap_remove_write(vm_page_t m) if ((m->flags & PG_FICTITIOUS) != 0 || (m->flags & PG_WRITEABLE) == 0) return; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) { va = pv->pv_va; @@ -4075,7 +4094,7 @@ pmap_ts_referenced(vm_page_t m) if (m->flags & PG_FICTITIOUS) return (rtval); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, pvn) { va = pv->pv_va; @@ -4147,7 +4166,7 @@ pmap_clear_modify(vm_page_t m) if ((m->flags & PG_FICTITIOUS) != 0) return; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) { va = pv->pv_va; @@ -4212,7 +4231,7 @@ pmap_clear_reference(vm_page_t m) if ((m->flags & PG_FICTITIOUS) != 0) return; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) { va = pv->pv_va; @@ -4696,7 +4715,7 @@ pmap_mincore(pmap_t pmap, vm_offset_t ad /* * Modified by someone else */ - vm_page_lock_queues(); + vm_page_lock_queues(); /* XXX */ if (m->dirty || pmap_is_modified(m)) val |= MINCORE_MODIFIED_OTHER; vm_page_unlock_queues(); @@ -4710,7 +4729,7 @@ pmap_mincore(pmap_t pmap, vm_offset_t ad /* * Referenced by someone else */ - vm_page_lock_queues(); + vm_page_lock_queues(); /* XXX */ if ((m->flags & PG_REFERENCED) || pmap_ts_referenced(m)) { val |= MINCORE_REFERENCED_OTHER; Modified: user/alc/pagelock/sys/ddb/db_watch.c ============================================================================== --- user/alc/pagelock/sys/ddb/db_watch.c Tue Feb 17 03:06:40 2009 (r188702) +++ user/alc/pagelock/sys/ddb/db_watch.c Tue Feb 17 03:58:15 2009 (r188703) @@ -238,7 +238,7 @@ db_set_watchpoints() for (watch = db_watchpoint_list; watch != 0; watch = watch->link) - pmap_protect(watch->map->pmap, + pmap_protect(watch->map->pmap, /* XXX */ trunc_page(watch->loaddr), round_page(watch->hiaddr), VM_PROT_READ); Modified: user/alc/pagelock/sys/kern/kern_subr.c ============================================================================== --- user/alc/pagelock/sys/kern/kern_subr.c Tue Feb 17 03:06:40 2009 (r188702) +++ user/alc/pagelock/sys/kern/kern_subr.c Tue Feb 17 03:58:15 2009 (r188703) @@ -116,8 +116,11 @@ retry: * object chain's first object, a physical page from a * backing object may be mapped read only. */ - if (uobject->backing_object != NULL) + if (uobject->backing_object != NULL) { + vm_object_lock_all(uobject->backing_object); pmap_remove(map->pmap, uaddr, uaddr + PAGE_SIZE); + vm_object_unlock_all(uobject->backing_object); + } vm_page_lock_queues(); } vm_page_insert(kern_pg, uobject, upindex); Modified: user/alc/pagelock/sys/kern/uipc_shm.c ============================================================================== --- user/alc/pagelock/sys/kern/uipc_shm.c Tue Feb 17 03:06:40 2009 (r188702) +++ user/alc/pagelock/sys/kern/uipc_shm.c Tue Feb 17 03:58:15 2009 (r188703) @@ -286,11 +286,9 @@ shm_dotruncate(struct shmfd *shmfd, off_ int size = PAGE_SIZE - base; pmap_zero_page_area(m, base, size); - vm_page_lock_queues(); vm_page_set_validclean(m, base, size); if (m->dirty != 0) m->dirty = VM_PAGE_BITS_ALL; - vm_page_unlock_queues(); } else if ((length & PAGE_MASK) && __predict_false(object->cache != NULL)) { vm_page_cache_free(object, OFF_TO_IDX(length), Modified: user/alc/pagelock/sys/kern/vfs_bio.c ============================================================================== --- user/alc/pagelock/sys/kern/vfs_bio.c Tue Feb 17 03:06:40 2009 (r188702) +++ user/alc/pagelock/sys/kern/vfs_bio.c Tue Feb 17 03:58:15 2009 (r188703) @@ -2309,7 +2309,6 @@ vfs_setdirty_locked_object(struct buf *b vm_offset_t boffset; vm_offset_t eoffset; - vm_page_lock_queues(); /* * test the pages to see if they have been modified directly * by users through the VM system. @@ -2335,7 +2334,6 @@ vfs_setdirty_locked_object(struct buf *b } eoffset = ((i + 1) << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK); - vm_page_unlock_queues(); /* * Fit it to the buffer. */ @@ -3162,7 +3160,6 @@ bufdone_finish(struct buf *bp) vm_object_t obj; int iosize; struct vnode *vp = bp->b_vp; - boolean_t are_queues_locked; obj = bp->b_bufobj->bo_object; @@ -3199,11 +3196,6 @@ bufdone_finish(struct buf *bp) !(bp->b_ioflags & BIO_ERROR)) { bp->b_flags |= B_CACHE; } - if (bp->b_iocmd == BIO_READ) { - vm_page_lock_queues(); - are_queues_locked = TRUE; - } else - are_queues_locked = FALSE; for (i = 0; i < bp->b_npages; i++) { int bogusflag = 0; int resid; @@ -3272,8 +3264,6 @@ bufdone_finish(struct buf *bp) foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK; iosize -= resid; } - if (are_queues_locked) - vm_page_unlock_queues(); vm_object_pip_wakeupn(obj, 0); VM_OBJECT_UNLOCK(obj); } @@ -3341,7 +3331,6 @@ vfs_page_set_valid(struct buf *bp, vm_oo { vm_ooffset_t soff, eoff; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); /* * Start and end offsets in buffer. eoff - soff may not cross a * page boundry or cross the end of the buffer. The end of the @@ -3466,7 +3455,6 @@ vfs_clean_pages(struct buf *bp) KASSERT(bp->b_offset != NOOFFSET, ("vfs_clean_pages: no buffer offset")); VM_OBJECT_LOCK(bp->b_bufobj->bo_object); - vm_page_lock_queues(); for (i = 0; i < bp->b_npages; i++) { m = bp->b_pages[i]; noff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK; @@ -3478,7 +3466,6 @@ vfs_clean_pages(struct buf *bp) /* vm_page_clear_dirty(m, foff & PAGE_MASK, eoff - foff); */ foff = noff; } - vm_page_unlock_queues(); VM_OBJECT_UNLOCK(bp->b_bufobj->bo_object); } @@ -3509,7 +3496,6 @@ vfs_bio_set_validclean(struct buf *bp, i n = PAGE_SIZE - (base & PAGE_MASK); VM_OBJECT_LOCK(bp->b_bufobj->bo_object); - vm_page_lock_queues(); for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) { m = bp->b_pages[i]; if (n > size) @@ -3519,7 +3505,6 @@ vfs_bio_set_validclean(struct buf *bp, i size -= n; n = PAGE_SIZE; } - vm_page_unlock_queues(); VM_OBJECT_UNLOCK(bp->b_bufobj->bo_object); } Modified: user/alc/pagelock/sys/security/mac/mac_process.c ============================================================================== --- user/alc/pagelock/sys/security/mac/mac_process.c Tue Feb 17 03:06:40 2009 (r188702) +++ user/alc/pagelock/sys/security/mac/mac_process.c Tue Feb 17 03:58:15 2009 (r188703) @@ -364,8 +364,10 @@ mac_proc_vm_revoke_recurse(struct thread vme->max_protection = 0; vme->protection = 0; } + vm_object_lock_all(vme->object.vm_object); pmap_protect(map->pmap, vme->start, vme->end, vme->protection & ~revokeperms); + vm_object_unlock_all(vme->object.vm_object); vm_map_simplify_entry(map, vme); } VFS_UNLOCK_GIANT(vfslocked); Modified: user/alc/pagelock/sys/vm/vm_fault.c ============================================================================== --- user/alc/pagelock/sys/vm/vm_fault.c Tue Feb 17 03:06:40 2009 (r188702) +++ user/alc/pagelock/sys/vm/vm_fault.c Tue Feb 17 03:58:15 2009 (r188703) @@ -919,7 +919,7 @@ vnode_locked: vm_page_zero_invalid(fs.m, TRUE); printf("Warning: page %p partially invalid on fault\n", fs.m); } - VM_OBJECT_UNLOCK(fs.object); + vm_object_lock_all(fs.object->backing_object); /* * Put this page into the physical map. We had to do the unlock above @@ -928,6 +928,7 @@ vnode_locked: * won't find it (yet). */ pmap_enter(fs.map->pmap, vaddr, fault_type, fs.m, prot, wired); + vm_object_unlock_all(fs.object); if (((fault_flags & VM_FAULT_WIRE_MASK) == 0) && (wired == 0)) { vm_fault_prefault(fs.map->pmap, vaddr, fs.entry); } @@ -1024,12 +1025,8 @@ vm_fault_prefault(pmap_t pmap, vm_offset } if (((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && (m->busy == 0) && - (m->flags & PG_FICTITIOUS) == 0) { - - vm_page_lock_queues(); + (m->flags & PG_FICTITIOUS) == 0) pmap_enter_quick(pmap, addr, m, entry->protection); - vm_page_unlock_queues(); - } VM_OBJECT_UNLOCK(lobject); } } Modified: user/alc/pagelock/sys/vm/vm_map.c ============================================================================== --- user/alc/pagelock/sys/vm/vm_map.c Tue Feb 17 03:06:40 2009 (r188702) +++ user/alc/pagelock/sys/vm/vm_map.c Tue Feb 17 03:58:15 2009 (r188703) @@ -1576,7 +1576,6 @@ vm_map_pmap_enter(vm_map_t map, vm_offse vm_offset_t start; vm_page_t p, p_start; vm_pindex_t psize, tmpidx; - boolean_t are_queues_locked; if ((prot & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0 || object == NULL) return; @@ -1600,7 +1599,6 @@ vm_map_pmap_enter(vm_map_t map, vm_offse psize = object->size - pindex; } - are_queues_locked = FALSE; start = 0; p_start = NULL; @@ -1635,25 +1633,15 @@ vm_map_pmap_enter(vm_map_t map, vm_offse p_start = p; } } else if (p_start != NULL) { - if (!are_queues_locked) { - are_queues_locked = TRUE; - vm_page_lock_queues(); - } pmap_enter_object(map->pmap, start, addr + ptoa(tmpidx), p_start, prot); p_start = NULL; } } if (p_start != NULL) { - if (!are_queues_locked) { - are_queues_locked = TRUE; - vm_page_lock_queues(); - } pmap_enter_object(map->pmap, start, addr + ptoa(psize), p_start, prot); } - if (are_queues_locked) - vm_page_unlock_queues(); unlock_return: VM_OBJECT_UNLOCK(object); } @@ -1722,12 +1710,15 @@ vm_map_protect(vm_map_t map, vm_offset_t * here. */ if (current->protection != old_prot) { + vm_object_lock_all(current->object.vm_object); + #define MASK(entry) (((entry)->eflags & MAP_ENTRY_COW) ? ~VM_PROT_WRITE : \ VM_PROT_ALL) pmap_protect(map->pmap, current->start, current->end, current->protection & MASK(current)); #undef MASK + vm_object_unlock_all(current->object.vm_object); } vm_map_simplify_entry(map, current); current = current->next; @@ -2335,9 +2326,6 @@ vm_map_sync( } } - if (invalidate) - pmap_remove(map->pmap, start, end); - /* * Make a second pass, cleaning/uncaching pages from the indicated * objects as we go. @@ -2362,6 +2350,11 @@ vm_map_sync( } else { object = current->object.vm_object; } + if (invalidate) { + vm_object_lock_all(object); + pmap_remove(map->pmap, start, start + size); + vm_object_unlock_all(object); + } vm_object_reference(object); last_timestamp = map->timestamp; vm_map_unlock_read(map); @@ -2524,7 +2517,12 @@ vm_map_delete(vm_map_t map, vm_offset_t vm_map_entry_unwire(map, entry); } - pmap_remove(map->pmap, entry->start, entry->end); + if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) { + vm_object_lock_all(entry->object.vm_object); + pmap_remove(map->pmap, entry->start, entry->end); + vm_object_unlock_all(entry->object.vm_object); + } else + panic("vm_map_delete: submap"); /* XXX */ /* * Delete the entry (which may delete the object) only after @@ -2631,10 +2629,14 @@ vm_map_copy_entry( * write-protected. */ if ((src_entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) { + vm_object_lock_all(src_entry->object.vm_object); + pmap_protect(src_map->pmap, src_entry->start, src_entry->end, src_entry->protection & ~VM_PROT_WRITE); + + vm_object_unlock_all(src_entry->object.vm_object); } /* @@ -2663,8 +2665,10 @@ vm_map_copy_entry( dst_entry->offset = 0; } + vm_object_lock_all(dst_entry->object.vm_object); pmap_copy(dst_map->pmap, src_map->pmap, dst_entry->start, dst_entry->end - dst_entry->start, src_entry->start); + vm_object_unlock_all(dst_entry->object.vm_object); } else { /* * Of course, wired down pages can't be set copy-on-write. @@ -2808,10 +2812,12 @@ vmspace_fork(struct vmspace *vm1) /* * Update the physical map */ + vm_object_lock_all(new_entry->object.vm_object); pmap_copy(new_map->pmap, old_map->pmap, new_entry->start, (old_entry->end - old_entry->start), old_entry->start); + vm_object_unlock_all(new_entry->object.vm_object); break; case VM_INHERIT_COPY: Modified: user/alc/pagelock/sys/vm/vm_object.c ============================================================================== --- user/alc/pagelock/sys/vm/vm_object.c Tue Feb 17 03:06:40 2009 (r188702) +++ user/alc/pagelock/sys/vm/vm_object.c Tue Feb 17 03:58:15 2009 (r188703) @@ -2010,6 +2010,32 @@ vm_object_coalesce(vm_object_t prev_obje return (TRUE); } +/* + * + */ +void +vm_object_lock_all(vm_object_t object) +{ + + while (object != NULL) { + VM_OBJECT_LOCK(object); + object = object->backing_object; + } +} + +/* + * + */ +void +vm_object_unlock_all(vm_object_t object) +{ + + while (object != NULL) { + VM_OBJECT_UNLOCK(object); + object = object->backing_object; + } +} + void vm_object_set_writeable_dirty(vm_object_t object) { Modified: user/alc/pagelock/sys/vm/vm_object.h ============================================================================== --- user/alc/pagelock/sys/vm/vm_object.h Tue Feb 17 03:06:40 2009 (r188702) +++ user/alc/pagelock/sys/vm/vm_object.h Tue Feb 17 03:58:15 2009 (r188703) @@ -205,6 +205,7 @@ void vm_object_destroy (vm_object_t); void vm_object_terminate (vm_object_t); void vm_object_set_writeable_dirty (vm_object_t); void vm_object_init (void); +void vm_object_lock_all(vm_object_t); void vm_object_page_clean (vm_object_t, vm_pindex_t, vm_pindex_t, boolean_t); void vm_object_page_remove (vm_object_t, vm_pindex_t, vm_pindex_t, boolean_t); void vm_object_reference (vm_object_t); @@ -213,6 +214,7 @@ void vm_object_shadow (vm_object_t *, vm void vm_object_split(vm_map_entry_t); void vm_object_sync(vm_object_t, vm_ooffset_t, vm_size_t, boolean_t, boolean_t); +void vm_object_unlock_all(vm_object_t); void vm_object_madvise (vm_object_t, vm_pindex_t, int, int); #endif /* _KERNEL */ Modified: user/alc/pagelock/sys/vm/vm_page.c ============================================================================== --- user/alc/pagelock/sys/vm/vm_page.c Tue Feb 17 03:06:40 2009 (r188702) +++ user/alc/pagelock/sys/vm/vm_page.c Tue Feb 17 03:58:15 2009 (r188703) @@ -566,6 +566,8 @@ vm_page_sleep(vm_page_t m, const char *m void vm_page_dirty(vm_page_t m) { + + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); KASSERT((m->flags & PG_CACHED) == 0, ("vm_page_dirty: page in cache!")); KASSERT(!VM_PAGE_IS_FREE(m), @@ -1594,7 +1596,7 @@ vm_page_try_to_free(vm_page_t m) (m->oflags & VPO_BUSY) || (m->flags & PG_UNMANAGED)) { return (0); } - pmap_remove_all(m); + pmap_remove_all(m); /* XXX */ if (m->dirty) return (0); vm_page_free(m); @@ -1741,6 +1743,7 @@ vm_page_dontneed(vm_page_t m) int head; mtx_assert(&vm_page_queue_mtx, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); dnw = ++dnweight; /* @@ -1870,7 +1873,6 @@ vm_page_set_validclean(vm_page_t m, int int frag; int endoff; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); if (size == 0) /* handle degenerate case */ return; @@ -1929,6 +1931,7 @@ void vm_page_clear_dirty(vm_page_t m, int base, int size) { + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); /* XXX */ mtx_assert(&vm_page_queue_mtx, MA_OWNED); m->dirty &= ~vm_page_bits(base, size); } @@ -2028,6 +2031,8 @@ vm_page_is_valid(vm_page_t m, int base, void vm_page_test_dirty(vm_page_t m) { + + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); if ((m->dirty != VM_PAGE_BITS_ALL) && pmap_is_modified(m)) { vm_page_dirty(m); } @@ -2118,6 +2123,7 @@ vm_page_cowsetup(vm_page_t m) { mtx_assert(&vm_page_queue_mtx, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); /* XXX */ if (m->cow == USHRT_MAX - 1) return (EBUSY); m->cow++; Modified: user/alc/pagelock/sys/vm/vm_pageout.c ============================================================================== --- user/alc/pagelock/sys/vm/vm_pageout.c Tue Feb 17 03:06:40 2009 (r188702) +++ user/alc/pagelock/sys/vm/vm_pageout.c Tue Feb 17 03:58:15 2009 (r188703) @@ -664,8 +664,7 @@ vm_pageout_map_deactivate_pages(map, des * table pages. */ if (desired == 0 && nothingwired) { - pmap_remove(vm_map_pmap(map), vm_map_min(map), - vm_map_max(map)); + pmap_remove_pages(vm_map_pmap(map)); } vm_map_unlock(map); } Modified: user/alc/pagelock/sys/vm/vnode_pager.c ============================================================================== --- user/alc/pagelock/sys/vm/vnode_pager.c Tue Feb 17 03:06:40 2009 (r188702) +++ user/alc/pagelock/sys/vm/vnode_pager.c Tue Feb 17 03:58:15 2009 (r188703) @@ -415,11 +415,9 @@ vnode_pager_setsize(vp, nsize) * bits. This would prevent bogus_page * replacement from working properly. */ - vm_page_lock_queues(); vm_page_set_validclean(m, base, size); if (m->dirty != 0) m->dirty = VM_PAGE_BITS_ALL; - vm_page_unlock_queues(); } else if ((nsize & PAGE_MASK) && __predict_false(object->cache != NULL)) { vm_page_cache_free(object, OFF_TO_IDX(nsize), @@ -545,23 +543,19 @@ vnode_pager_input_smlfs(object, m) break; VM_OBJECT_LOCK(object); - vm_page_lock_queues(); vm_page_set_validclean(m, (i * bsize) & PAGE_MASK, bsize); - vm_page_unlock_queues(); VM_OBJECT_UNLOCK(object); } else { VM_OBJECT_LOCK(object); - vm_page_lock_queues(); vm_page_set_validclean(m, (i * bsize) & PAGE_MASK, bsize); - vm_page_unlock_queues(); VM_OBJECT_UNLOCK(object); bzero((caddr_t)sf_buf_kva(sf) + i * bsize, bsize); } } sf_buf_free(sf); - vm_page_lock_queues(); + VM_OBJECT_LOCK(object); pmap_clear_modify(m); - vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(object); if (error) { return VM_PAGER_ERROR; } @@ -630,10 +624,8 @@ vnode_pager_input_old(object, m) VM_OBJECT_LOCK(object); } - vm_page_lock_queues(); pmap_clear_modify(m); vm_page_undirty(m); - vm_page_unlock_queues(); if (!error) m->valid = VM_PAGE_BITS_ALL; return error ? VM_PAGER_ERROR : VM_PAGER_OK;