From owner-svn-src-head@FreeBSD.ORG Fri Apr 30 00:46:44 2010 Return-Path: Delivered-To: svn-src-head@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [69.147.83.52]) by hub.freebsd.org (Postfix) with ESMTP id 7F638106564A; Fri, 30 Apr 2010 00:46:44 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [69.147.83.44]) by mx1.freebsd.org (Postfix) with ESMTP id 69FB78FC08; Fri, 30 Apr 2010 00:46:44 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id o3U0kiah008451; Fri, 30 Apr 2010 00:46:44 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id o3U0khG1008413; Fri, 30 Apr 2010 00:46:43 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <201004300046.o3U0khG1008413@svn.freebsd.org> From: Kip Macy Date: Fri, 30 Apr 2010 00:46:43 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r207410 - in head: . sys/amd64/amd64 sys/amd64/include sys/arm/arm sys/arm/include sys/dev/drm sys/i386/i386 sys/i386/include sys/i386/xen sys/ia64/ia64 sys/ia64/include sys/kern sys/mi... X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 30 Apr 2010 00:46:44 -0000 Author: kmacy Date: Fri Apr 30 00:46:43 2010 New Revision: 207410 URL: http://svn.freebsd.org/changeset/base/207410 Log: On Alan's advice, rather than do a wholesale conversion on a single architecture from page queue lock to a hashed array of page locks (based on a patch by Jeff Roberson), I've implemented page lock support in the MI code and have only moved vm_page's hold_count out from under page queue mutex to page lock. This changes pmap_extract_and_hold on all pmaps. Supported by: Bitgravity Inc. Discussed with: alc, jeffr, and kib Modified: head/UPDATING head/sys/amd64/amd64/pmap.c head/sys/amd64/include/pmap.h head/sys/amd64/include/vmparam.h head/sys/arm/arm/pmap.c head/sys/arm/include/pmap.h head/sys/dev/drm/via_dmablit.c head/sys/i386/i386/pmap.c head/sys/i386/include/pmap.h head/sys/i386/xen/pmap.c head/sys/ia64/ia64/pmap.c head/sys/ia64/include/pmap.h head/sys/kern/kern_exec.c head/sys/kern/subr_witness.c head/sys/kern/sys_pipe.c head/sys/kern/sys_process.c head/sys/kern/uipc_cow.c head/sys/kern/vfs_bio.c head/sys/mips/include/pmap.h head/sys/mips/mips/pmap.c head/sys/net/bpf_zerocopy.c head/sys/powerpc/aim/mmu_oea.c head/sys/powerpc/aim/mmu_oea64.c head/sys/powerpc/booke/pmap.c head/sys/powerpc/include/pmap.h head/sys/sparc64/include/pmap.h head/sys/sparc64/sparc64/pmap.c head/sys/sun4v/include/pmap.h head/sys/sun4v/sun4v/pmap.c head/sys/sys/param.h head/sys/vm/device_pager.c head/sys/vm/sg_pager.c head/sys/vm/swap_pager.c head/sys/vm/uma_core.c head/sys/vm/vm_contig.c head/sys/vm/vm_fault.c head/sys/vm/vm_glue.c head/sys/vm/vm_kern.c head/sys/vm/vm_mmap.c head/sys/vm/vm_object.c head/sys/vm/vm_page.c head/sys/vm/vm_page.h head/sys/vm/vm_pageout.c head/sys/vm/vm_param.h head/sys/vm/vnode_pager.c Modified: head/UPDATING ============================================================================== --- head/UPDATING Fri Apr 30 00:34:00 2010 (r207409) +++ head/UPDATING Fri Apr 30 00:46:43 2010 (r207410) @@ -22,6 +22,14 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 9. machines to maximize performance. (To disable malloc debugging, run ln -s aj /etc/malloc.conf.) + +20100429: + 'vm_page's are now hashed by physical address to an array of mutexes. + Currently this is only used to serialize access to hold_count. Over + time the page queue mutex will be peeled away. This changes the size + of pmap on every architecture. And requires all callers of vm_page_hold + and vm_page_unhold to be updated. + 20100402: WITH_CTF can now be specified in src.conf (not recommended, there are some problems with static executables), make.conf (would also Modified: head/sys/amd64/amd64/pmap.c ============================================================================== --- head/sys/amd64/amd64/pmap.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/amd64/amd64/pmap.c Fri Apr 30 00:46:43 2010 (r207410) @@ -793,7 +793,6 @@ static u_long pmap_pdpe_demotions; SYSCTL_ULONG(_vm_pmap_pdpe, OID_AUTO, demotions, CTLFLAG_RD, &pmap_pdpe_demotions, 0, "1GB page demotions"); - /*************************************************** * Low level helper routines..... ***************************************************/ @@ -1200,15 +1199,20 @@ pmap_extract_and_hold(pmap_t pmap, vm_of { pd_entry_t pde, *pdep; pt_entry_t pte; + vm_paddr_t pa; vm_page_t m; + pa = 0; m = NULL; - vm_page_lock_queues(); PMAP_LOCK(pmap); +retry: pdep = pmap_pde(pmap, va); if (pdep != NULL && (pde = *pdep)) { if (pde & PG_PS) { if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) { + if (vm_page_pa_tryrelock(pmap, (pde & PG_PS_FRAME) | + (va & PDRMASK), &pa)) + goto retry; m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) | (va & PDRMASK)); vm_page_hold(m); @@ -1217,12 +1221,14 @@ pmap_extract_and_hold(pmap_t pmap, vm_of pte = *pmap_pde_to_pte(pdep, va); if ((pte & PG_V) && ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) { + if (vm_page_pa_tryrelock(pmap, pte & PG_FRAME, &pa)) + goto retry; m = PHYS_TO_VM_PAGE(pte & PG_FRAME); vm_page_hold(m); } } } - vm_page_unlock_queues(); + PA_UNLOCK_COND(pa); PMAP_UNLOCK(pmap); return (m); } @@ -3143,9 +3149,8 @@ pmap_enter(pmap_t pmap, vm_offset_t va, * In the case that a page table page is not * resident, we are creating it here. */ - if (va < VM_MAXUSER_ADDRESS) { + if (va < VM_MAXUSER_ADDRESS) mpte = pmap_allocpte(pmap, va, M_WAITOK); - } pde = pmap_pde(pmap, va); if (pde != NULL && (*pde & PG_V) != 0) { @@ -3393,7 +3398,7 @@ pmap_enter_object(pmap_t pmap, vm_offset mpte); m = TAILQ_NEXT(m, listq); } - PMAP_UNLOCK(pmap); + PMAP_UNLOCK(pmap); } /* Modified: head/sys/amd64/include/pmap.h ============================================================================== --- head/sys/amd64/include/pmap.h Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/amd64/include/pmap.h Fri Apr 30 00:46:43 2010 (r207410) @@ -245,6 +245,8 @@ struct pmap { pml4_entry_t *pm_pml4; /* KVA of level 4 page table */ TAILQ_HEAD(,pv_chunk) pm_pvchunk; /* list of mappings in pmap */ u_int pm_active; /* active on cpus */ + uint32_t pm_gen_count; /* generation count (pmap lock dropped) */ + u_int pm_retries; /* spare u_int here due to padding */ struct pmap_statistics pm_stats; /* pmap statistics */ vm_page_t pm_root; /* spare page table pages */ Modified: head/sys/amd64/include/vmparam.h ============================================================================== --- head/sys/amd64/include/vmparam.h Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/amd64/include/vmparam.h Fri Apr 30 00:46:43 2010 (r207410) @@ -145,6 +145,10 @@ #define VM_LEVEL_0_ORDER 9 #endif +#ifdef SMP +#define PA_LOCK_COUNT 256 +#endif + /* * Virtual addresses of things. Derived from the page directory and * page table indexes from pmap.h for precision. Modified: head/sys/arm/arm/pmap.c ============================================================================== --- head/sys/arm/arm/pmap.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/arm/arm/pmap.c Fri Apr 30 00:46:43 2010 (r207410) @@ -3740,13 +3740,14 @@ pmap_extract_and_hold(pmap_t pmap, vm_of struct l2_dtable *l2; pd_entry_t l1pd; pt_entry_t *ptep, pte; - vm_paddr_t pa; + vm_paddr_t pa, paddr; vm_page_t m = NULL; u_int l1idx; l1idx = L1_IDX(va); + paddr = 0; - vm_page_lock_queues(); PMAP_LOCK(pmap); +retry: l1pd = pmap->pm_l1->l1_kva[l1idx]; if (l1pte_section_p(l1pd)) { /* @@ -3758,6 +3759,8 @@ pmap_extract_and_hold(pmap_t pmap, vm_of pa = (l1pd & L1_SUP_FRAME) | (va & L1_SUP_OFFSET); else pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); + if (vm_page_pa_tryrelock(pmap, pa & PG_FRAME, &paddr)) + goto retry; if (l1pd & L1_S_PROT_W || (prot & VM_PROT_WRITE) == 0) { m = PHYS_TO_VM_PAGE(pa); vm_page_hold(m); @@ -3774,7 +3777,6 @@ pmap_extract_and_hold(pmap_t pmap, vm_of if (l2 == NULL || (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { PMAP_UNLOCK(pmap); - vm_page_unlock_queues(); return (NULL); } @@ -3783,7 +3785,6 @@ pmap_extract_and_hold(pmap_t pmap, vm_of if (pte == 0) { PMAP_UNLOCK(pmap); - vm_page_unlock_queues(); return (NULL); } if (pte & L2_S_PROT_W || (prot & VM_PROT_WRITE) == 0) { @@ -3796,13 +3797,15 @@ pmap_extract_and_hold(pmap_t pmap, vm_of pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); break; } + if (vm_page_pa_tryrelock(pmap, pa & PG_FRAME, &paddr)) + goto retry; m = PHYS_TO_VM_PAGE(pa); vm_page_hold(m); } } PMAP_UNLOCK(pmap); - vm_page_unlock_queues(); + PA_UNLOCK_COND(paddr); return (m); } Modified: head/sys/arm/include/pmap.h ============================================================================== --- head/sys/arm/include/pmap.h Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/arm/include/pmap.h Fri Apr 30 00:46:43 2010 (r207410) @@ -134,6 +134,8 @@ struct pmap { struct l1_ttable *pm_l1; struct l2_dtable *pm_l2[L2_SIZE]; pd_entry_t *pm_pdir; /* KVA of page directory */ + uint32_t pm_gen_count; /* generation count (pmap lock dropped) */ + u_int pm_retries; int pm_active; /* active on cpus */ struct pmap_statistics pm_stats; /* pmap statictics */ TAILQ_HEAD(,pv_entry) pm_pvlist; /* list of mappings in pmap */ Modified: head/sys/dev/drm/via_dmablit.c ============================================================================== --- head/sys/dev/drm/via_dmablit.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/dev/drm/via_dmablit.c Fri Apr 30 00:46:43 2010 (r207410) @@ -248,10 +248,12 @@ via_lock_all_dma_pages(drm_via_sg_info_t (vm_offset_t)xfer->mem_addr + IDX_TO_OFF(i), VM_PROT_RW); if (m == NULL) break; + vm_page_lock(m); vm_page_lock_queues(); vm_page_wire(m); vm_page_unhold(m); vm_page_unlock_queues(); + vm_page_unlock(m); vsg->pages[i] = m; } vsg->state = dr_via_pages_locked; Modified: head/sys/i386/i386/pmap.c ============================================================================== --- head/sys/i386/i386/pmap.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/i386/i386/pmap.c Fri Apr 30 00:46:43 2010 (r207410) @@ -1346,14 +1346,19 @@ pmap_extract_and_hold(pmap_t pmap, vm_of pd_entry_t pde; pt_entry_t pte; vm_page_t m; + vm_paddr_t pa; + pa = 0; m = NULL; - vm_page_lock_queues(); PMAP_LOCK(pmap); +retry: pde = *pmap_pde(pmap, va); if (pde != 0) { if (pde & PG_PS) { if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) { + if (vm_page_pa_tryrelock(pmap, (pde & PG_PS_FRAME) | + (va & PDRMASK), &pa)) + goto retry; m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) | (va & PDRMASK)); vm_page_hold(m); @@ -1363,13 +1368,15 @@ pmap_extract_and_hold(pmap_t pmap, vm_of pte = *pmap_pte_quick(pmap, va); if (pte != 0 && ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) { + if (vm_page_pa_tryrelock(pmap, pte & PG_FRAME, &pa)) + goto retry; m = PHYS_TO_VM_PAGE(pte & PG_FRAME); vm_page_hold(m); } sched_unpin(); } } - vm_page_unlock_queues(); + PA_UNLOCK_COND(pa); PMAP_UNLOCK(pmap); return (m); } Modified: head/sys/i386/include/pmap.h ============================================================================== --- head/sys/i386/include/pmap.h Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/i386/include/pmap.h Fri Apr 30 00:46:43 2010 (r207410) @@ -420,11 +420,14 @@ struct pmap { u_int pm_active; /* active on cpus */ struct pmap_statistics pm_stats; /* pmap statistics */ LIST_ENTRY(pmap) pm_list; /* List of all pmaps */ + uint32_t pm_gen_count; /* generation count (pmap lock dropped) */ + u_int pm_retries; #ifdef PAE pdpt_entry_t *pm_pdpt; /* KVA of page director pointer table */ #endif vm_page_t pm_root; /* spare page table pages */ + }; typedef struct pmap *pmap_t; Modified: head/sys/i386/xen/pmap.c ============================================================================== --- head/sys/i386/xen/pmap.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/i386/xen/pmap.c Fri Apr 30 00:46:43 2010 (r207410) @@ -1219,14 +1219,19 @@ pmap_extract_and_hold(pmap_t pmap, vm_of pd_entry_t pde; pt_entry_t pte; vm_page_t m; + vm_paddr_t pa; + pa = 0; m = NULL; - vm_page_lock_queues(); PMAP_LOCK(pmap); +retry: pde = PT_GET(pmap_pde(pmap, va)); if (pde != 0) { if (pde & PG_PS) { if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) { + if (vm_page_pa_tryrelock(pmap, (pde & PG_PS_FRAME) | + (va & PDRMASK), &pa)) + goto retry; m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) | (va & PDRMASK)); vm_page_hold(m); @@ -1238,13 +1243,15 @@ pmap_extract_and_hold(pmap_t pmap, vm_of PT_SET_MA(PADDR1, 0); if ((pte & PG_V) && ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) { + if (vm_page_pa_tryrelock(pmap, pte & PG_FRAME, &pa)) + goto retry; m = PHYS_TO_VM_PAGE(pte & PG_FRAME); vm_page_hold(m); } sched_unpin(); } } - vm_page_unlock_queues(); + PA_UNLOCK_COND(pa); PMAP_UNLOCK(pmap); return (m); } Modified: head/sys/ia64/ia64/pmap.c ============================================================================== --- head/sys/ia64/ia64/pmap.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/ia64/ia64/pmap.c Fri Apr 30 00:46:43 2010 (r207410) @@ -1028,18 +1028,22 @@ pmap_extract_and_hold(pmap_t pmap, vm_of struct ia64_lpte *pte; pmap_t oldpmap; vm_page_t m; + vm_paddr_t pa; + pa = 0; m = NULL; - vm_page_lock_queues(); PMAP_LOCK(pmap); oldpmap = pmap_switch(pmap); +retry: pte = pmap_find_vhpt(va); if (pte != NULL && pmap_present(pte) && (pmap_prot(pte) & prot) == prot) { m = PHYS_TO_VM_PAGE(pmap_ppn(pte)); + if (vm_page_pa_tryrelock(pmap, pmap_ppn(pte), &pa)) + goto retry; vm_page_hold(m); } - vm_page_unlock_queues(); + PA_UNLOCK_COND(pa); pmap_switch(oldpmap); PMAP_UNLOCK(pmap); return (m); Modified: head/sys/ia64/include/pmap.h ============================================================================== --- head/sys/ia64/include/pmap.h Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/ia64/include/pmap.h Fri Apr 30 00:46:43 2010 (r207410) @@ -77,6 +77,8 @@ struct pmap { TAILQ_HEAD(,pv_entry) pm_pvlist; /* list of mappings in pmap */ u_int32_t pm_rid[5]; /* base RID for pmap */ struct pmap_statistics pm_stats; /* pmap statistics */ + uint32_t pm_gen_count; /* generation count (pmap lock dropped) */ + u_int pm_retries; }; typedef struct pmap *pmap_t; Modified: head/sys/kern/kern_exec.c ============================================================================== --- head/sys/kern/kern_exec.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/kern/kern_exec.c Fri Apr 30 00:46:43 2010 (r207410) @@ -957,9 +957,9 @@ exec_map_first_page(imgp) return (EIO); } } - vm_page_lock_queues(); + vm_page_lock(ma[0]); vm_page_hold(ma[0]); - vm_page_unlock_queues(); + vm_page_unlock(ma[0]); vm_page_wakeup(ma[0]); VM_OBJECT_UNLOCK(object); @@ -979,9 +979,9 @@ exec_unmap_first_page(imgp) m = sf_buf_page(imgp->firstpage); sf_buf_free(imgp->firstpage); imgp->firstpage = NULL; - vm_page_lock_queues(); + vm_page_lock(m); vm_page_unhold(m); - vm_page_unlock_queues(); + vm_page_unlock(m); } } Modified: head/sys/kern/subr_witness.c ============================================================================== --- head/sys/kern/subr_witness.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/kern/subr_witness.c Fri Apr 30 00:46:43 2010 (r207410) @@ -597,6 +597,15 @@ static struct witness_order_list_entry o { "cdev", &lock_class_mtx_sleep }, { NULL, NULL }, /* + * VM + * + */ + { "vm object", &lock_class_mtx_sleep }, + { "page lock", &lock_class_mtx_sleep }, + { "vm page queue mutex", &lock_class_mtx_sleep }, + { "pmap", &lock_class_mtx_sleep }, + { NULL, NULL }, + /* * kqueue/VFS interaction */ { "kqueue", &lock_class_mtx_sleep }, Modified: head/sys/kern/sys_pipe.c ============================================================================== --- head/sys/kern/sys_pipe.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/kern/sys_pipe.c Fri Apr 30 00:46:43 2010 (r207410) @@ -773,10 +773,12 @@ pipe_build_write_buffer(wpipe, uio) */ race: if (vm_fault_quick((caddr_t)addr, VM_PROT_READ) < 0) { - vm_page_lock_queues(); - for (j = 0; j < i; j++) + + for (j = 0; j < i; j++) { + vm_page_lock(wpipe->pipe_map.ms[j]); vm_page_unhold(wpipe->pipe_map.ms[j]); - vm_page_unlock_queues(); + vm_page_unlock(wpipe->pipe_map.ms[j]); + } return (EFAULT); } wpipe->pipe_map.ms[i] = pmap_extract_and_hold(pmap, addr, @@ -816,11 +818,11 @@ pipe_destroy_write_buffer(wpipe) int i; PIPE_LOCK_ASSERT(wpipe, MA_OWNED); - vm_page_lock_queues(); for (i = 0; i < wpipe->pipe_map.npages; i++) { + vm_page_lock(wpipe->pipe_map.ms[i]); vm_page_unhold(wpipe->pipe_map.ms[i]); + vm_page_unlock(wpipe->pipe_map.ms[i]); } - vm_page_unlock_queues(); wpipe->pipe_map.npages = 0; } Modified: head/sys/kern/sys_process.c ============================================================================== --- head/sys/kern/sys_process.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/kern/sys_process.c Fri Apr 30 00:46:43 2010 (r207410) @@ -328,9 +328,9 @@ proc_rwmem(struct proc *p, struct uio *u /* * Hold the page in memory. */ - vm_page_lock_queues(); + vm_page_lock(m); vm_page_hold(m); - vm_page_unlock_queues(); + vm_page_unlock(m); /* * We're done with tmap now. @@ -349,9 +349,9 @@ proc_rwmem(struct proc *p, struct uio *u /* * Release the page. */ - vm_page_lock_queues(); + vm_page_lock(m); vm_page_unhold(m); - vm_page_unlock_queues(); + vm_page_unlock(m); } while (error == 0 && uio->uio_resid > 0); Modified: head/sys/kern/uipc_cow.c ============================================================================== --- head/sys/kern/uipc_cow.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/kern/uipc_cow.c Fri Apr 30 00:46:43 2010 (r207410) @@ -128,10 +128,12 @@ socow_setup(struct mbuf *m0, struct uio /* * set up COW */ + vm_page_lock(pp); vm_page_lock_queues(); if (vm_page_cowsetup(pp) != 0) { vm_page_unhold(pp); vm_page_unlock_queues(); + vm_page_unlock(pp); return (0); } @@ -141,7 +143,7 @@ socow_setup(struct mbuf *m0, struct uio vm_page_wire(pp); vm_page_unhold(pp); vm_page_unlock_queues(); - + vm_page_unlock(pp); /* * Allocate an sf buf */ Modified: head/sys/kern/vfs_bio.c ============================================================================== --- head/sys/kern/vfs_bio.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/kern/vfs_bio.c Fri Apr 30 00:46:43 2010 (r207410) @@ -3860,12 +3860,12 @@ vmapbuf(struct buf *bp) retry: if (vm_fault_quick(addr >= bp->b_data ? addr : bp->b_data, prot) < 0) { - vm_page_lock_queues(); for (i = 0; i < pidx; ++i) { + vm_page_lock(bp->b_pages[i]); vm_page_unhold(bp->b_pages[i]); + vm_page_unlock(bp->b_pages[i]); bp->b_pages[i] = NULL; } - vm_page_unlock_queues(); return(-1); } m = pmap_extract_and_hold(pmap, (vm_offset_t)addr, prot); @@ -3896,11 +3896,12 @@ vunmapbuf(struct buf *bp) npages = bp->b_npages; pmap_qremove(trunc_page((vm_offset_t)bp->b_data), npages); - vm_page_lock_queues(); - for (pidx = 0; pidx < npages; pidx++) + for (pidx = 0; pidx < npages; pidx++) { + vm_page_lock(bp->b_pages[pidx]); vm_page_unhold(bp->b_pages[pidx]); - vm_page_unlock_queues(); - + vm_page_unlock(bp->b_pages[pidx]); + } + bp->b_data = bp->b_saveaddr; } Modified: head/sys/mips/include/pmap.h ============================================================================== --- head/sys/mips/include/pmap.h Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/mips/include/pmap.h Fri Apr 30 00:46:43 2010 (r207410) @@ -88,6 +88,8 @@ struct pmap { pd_entry_t *pm_segtab; /* KVA of segment table */ TAILQ_HEAD(, pv_entry) pm_pvlist; /* list of mappings in * pmap */ + uint32_t pm_gen_count; /* generation count (pmap lock dropped) */ + u_int pm_retries; int pm_active; /* active on cpus */ struct { u_int32_t asid:ASID_BITS; /* TLB address space tag */ Modified: head/sys/mips/mips/pmap.c ============================================================================== --- head/sys/mips/mips/pmap.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/mips/mips/pmap.c Fri Apr 30 00:46:43 2010 (r207410) @@ -147,7 +147,6 @@ unsigned pmap_max_asid; /* max ASID sup #define PMAP_ASID_RESERVED 0 - vm_offset_t kernel_vm_end; static struct tlb tlbstash[MAXCPU][MIPS_MAX_TLB_ENTRIES]; @@ -710,18 +709,22 @@ pmap_extract_and_hold(pmap_t pmap, vm_of { pt_entry_t pte; vm_page_t m; + vm_paddr_t pa; m = NULL; - vm_page_lock_queues(); + pa = 0; PMAP_LOCK(pmap); - +retry: pte = *pmap_pte(pmap, va); if (pte != 0 && pmap_pte_v(&pte) && ((pte & PTE_RW) || (prot & VM_PROT_WRITE) == 0)) { + if (vm_page_pa_tryrelock(pmap, mips_tlbpfn_to_paddr(pte), &pa)) + goto retry; + m = PHYS_TO_VM_PAGE(mips_tlbpfn_to_paddr(pte)); vm_page_hold(m); } - vm_page_unlock_queues(); + PA_UNLOCK_COND(pa); PMAP_UNLOCK(pmap); return (m); } Modified: head/sys/net/bpf_zerocopy.c ============================================================================== --- head/sys/net/bpf_zerocopy.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/net/bpf_zerocopy.c Fri Apr 30 00:46:43 2010 (r207410) @@ -168,10 +168,12 @@ zbuf_sfbuf_get(struct vm_map *map, vm_of VM_PROT_WRITE); if (pp == NULL) return (NULL); + vm_page_lock(pp); vm_page_lock_queues(); vm_page_wire(pp); vm_page_unhold(pp); vm_page_unlock_queues(); + vm_page_unlock(pp); sf = sf_buf_alloc(pp, SFB_NOWAIT); if (sf == NULL) { zbuf_page_free(pp); Modified: head/sys/powerpc/aim/mmu_oea.c ============================================================================== --- head/sys/powerpc/aim/mmu_oea.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/powerpc/aim/mmu_oea.c Fri Apr 30 00:46:43 2010 (r207410) @@ -1241,18 +1241,22 @@ moea_extract_and_hold(mmu_t mmu, pmap_t { struct pvo_entry *pvo; vm_page_t m; - + vm_paddr_t pa; + m = NULL; - vm_page_lock_queues(); + pa = 0; PMAP_LOCK(pmap); +retry: pvo = moea_pvo_find_va(pmap, va & ~ADDR_POFF, NULL); if (pvo != NULL && (pvo->pvo_pte.pte.pte_hi & PTE_VALID) && ((pvo->pvo_pte.pte.pte_lo & PTE_PP) == PTE_RW || (prot & VM_PROT_WRITE) == 0)) { + if (vm_page_pa_tryrelock(pmap, pvo->pvo_pte.pte.pte_lo & PTE_RPGN, &pa)) + goto retry; m = PHYS_TO_VM_PAGE(pvo->pvo_pte.pte.pte_lo & PTE_RPGN); vm_page_hold(m); } - vm_page_unlock_queues(); + PA_UNLOCK_COND(pa); PMAP_UNLOCK(pmap); return (m); } Modified: head/sys/powerpc/aim/mmu_oea64.c ============================================================================== --- head/sys/powerpc/aim/mmu_oea64.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/powerpc/aim/mmu_oea64.c Fri Apr 30 00:46:43 2010 (r207410) @@ -1374,18 +1374,23 @@ moea64_extract_and_hold(mmu_t mmu, pmap_ { struct pvo_entry *pvo; vm_page_t m; + vm_paddr_t pa; m = NULL; - vm_page_lock_queues(); + pa = 0; PMAP_LOCK(pmap); +retry: pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF, NULL); if (pvo != NULL && (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) && ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) == LPTE_RW || (prot & VM_PROT_WRITE) == 0)) { + if (vm_page_pa_tryrelock(pmap, + pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN, &pa)) + goto retry; m = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN); vm_page_hold(m); } - vm_page_unlock_queues(); + PA_UNLOCK_COND(pa); PMAP_UNLOCK(pmap); return (m); } Modified: head/sys/powerpc/booke/pmap.c ============================================================================== --- head/sys/powerpc/booke/pmap.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/powerpc/booke/pmap.c Fri Apr 30 00:46:43 2010 (r207410) @@ -2034,11 +2034,12 @@ mmu_booke_extract_and_hold(mmu_t mmu, pm pte_t *pte; vm_page_t m; uint32_t pte_wbit; - + vm_paddr_t pa; + m = NULL; - vm_page_lock_queues(); + pa = 0; PMAP_LOCK(pmap); - +retry: pte = pte_find(mmu, pmap, va); if ((pte != NULL) && PTE_ISVALID(pte)) { if (pmap == kernel_pmap) @@ -2047,12 +2048,14 @@ mmu_booke_extract_and_hold(mmu_t mmu, pm pte_wbit = PTE_UW; if ((pte->flags & pte_wbit) || ((prot & VM_PROT_WRITE) == 0)) { + if (vm_page_pa_tryrelock(pmap, PTE_PA(pte), &pa)) + goto retry; m = PHYS_TO_VM_PAGE(PTE_PA(pte)); vm_page_hold(m); } } - vm_page_unlock_queues(); + PA_UNLOCK_COND(pa); PMAP_UNLOCK(pmap); return (m); } Modified: head/sys/powerpc/include/pmap.h ============================================================================== --- head/sys/powerpc/include/pmap.h Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/powerpc/include/pmap.h Fri Apr 30 00:46:43 2010 (r207410) @@ -88,6 +88,8 @@ struct pmap { struct mtx pm_mtx; u_int pm_sr[16]; u_int pm_active; + uint32_t pm_gen_count; /* generation count (pmap lock dropped) */ + u_int pm_retries; u_int pm_context; struct pmap *pmap_phys; Modified: head/sys/sparc64/include/pmap.h ============================================================================== --- head/sys/sparc64/include/pmap.h Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/sparc64/include/pmap.h Fri Apr 30 00:46:43 2010 (r207410) @@ -62,6 +62,8 @@ struct pmap { struct tte *pm_tsb; vm_object_t pm_tsb_obj; u_int pm_active; + uint32_t pm_gen_count; /* generation count (pmap lock dropped) */ + u_int pm_retries; u_int pm_context[MAXCPU]; struct pmap_statistics pm_stats; }; Modified: head/sys/sparc64/sparc64/pmap.c ============================================================================== --- head/sys/sparc64/sparc64/pmap.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/sparc64/sparc64/pmap.c Fri Apr 30 00:46:43 2010 (r207410) @@ -694,13 +694,17 @@ pmap_extract_and_hold(pmap_t pm, vm_offs { struct tte *tp; vm_page_t m; + vm_paddr_t pa; m = NULL; - vm_page_lock_queues(); + pa = 0; + PMAP_LOCK(pm); +retry: if (pm == kernel_pmap) { if (va >= VM_MIN_DIRECT_ADDRESS) { tp = NULL; m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS(va)); + (void)vm_page_pa_tryrelock(pm, TLB_DIRECT_TO_PHYS(va), &pa); vm_page_hold(m); } else { tp = tsb_kvtotte(va); @@ -708,17 +712,17 @@ pmap_extract_and_hold(pmap_t pm, vm_offs tp = NULL; } } else { - PMAP_LOCK(pm); tp = tsb_tte_lookup(pm, va); } if (tp != NULL && ((tp->tte_data & TD_SW) || (prot & VM_PROT_WRITE) == 0)) { + if (vm_page_pa_tryrelock(pm, TTE_GET_PA(tp), &pa)) + goto retry; m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); vm_page_hold(m); } - vm_page_unlock_queues(); - if (pm != kernel_pmap) - PMAP_UNLOCK(pm); + PA_UNLOCK_COND(pa); + PMAP_UNLOCK(pm); return (m); } Modified: head/sys/sun4v/include/pmap.h ============================================================================== --- head/sys/sun4v/include/pmap.h Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/sun4v/include/pmap.h Fri Apr 30 00:46:43 2010 (r207410) @@ -75,6 +75,8 @@ struct pmap { struct tte_hash *pm_hash; TAILQ_HEAD(,pv_entry) pm_pvlist; /* list of mappings in pmap */ struct hv_tsb_info pm_tsb; + uint32_t pm_gen_count; /* generation count (pmap lock dropped) */ + u_int pm_retries; pmap_cpumask_t pm_active; /* mask of cpus currently using pmap */ pmap_cpumask_t pm_tlbactive; /* mask of cpus that have used this pmap */ struct pmap_statistics pm_stats; Modified: head/sys/sun4v/sun4v/pmap.c ============================================================================== --- head/sys/sun4v/sun4v/pmap.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/sun4v/sun4v/pmap.c Fri Apr 30 00:46:43 2010 (r207410) @@ -1275,17 +1275,21 @@ pmap_extract_and_hold(pmap_t pmap, vm_of { tte_t tte_data; vm_page_t m; + vm_paddr_t pa; m = NULL; - vm_page_lock_queues(); + pa = 0; PMAP_LOCK(pmap); +retry: tte_data = tte_hash_lookup(pmap->pm_hash, va); if (tte_data != 0 && ((tte_data & VTD_SW_W) || (prot & VM_PROT_WRITE) == 0)) { + if (vm_page_pa_tryrelock(pmap, TTE_GET_PA(tte_data), &pa)) + goto retry; m = PHYS_TO_VM_PAGE(TTE_GET_PA(tte_data)); vm_page_hold(m); } - vm_page_unlock_queues(); + PA_UNLOCK_COND(pa); PMAP_UNLOCK(pmap); return (m); Modified: head/sys/sys/param.h ============================================================================== --- head/sys/sys/param.h Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/sys/param.h Fri Apr 30 00:46:43 2010 (r207410) @@ -58,7 +58,7 @@ * in the range 5 to 9. */ #undef __FreeBSD_version -#define __FreeBSD_version 900010 /* Master, propagated to newvers */ +#define __FreeBSD_version 900011 /* Master, propagated to newvers */ #ifndef LOCORE #include Modified: head/sys/vm/device_pager.c ============================================================================== --- head/sys/vm/device_pager.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/vm/device_pager.c Fri Apr 30 00:46:43 2010 (r207410) @@ -251,12 +251,16 @@ dev_pager_getpages(object, m, count, req VM_OBJECT_LOCK(object); dev_pager_updatefake(page, paddr, memattr); if (count > 1) { - vm_page_lock_queues(); + for (i = 0; i < count; i++) { - if (i != reqpage) + if (i != reqpage) { + vm_page_lock(m[i]); + vm_page_lock_queues(); vm_page_free(m[i]); + vm_page_unlock_queues(); + vm_page_unlock(m[i]); + } } - vm_page_unlock_queues(); } } else { /* @@ -266,10 +270,13 @@ dev_pager_getpages(object, m, count, req page = dev_pager_getfake(paddr, memattr); VM_OBJECT_LOCK(object); TAILQ_INSERT_TAIL(&object->un_pager.devp.devp_pglist, page, pageq); - vm_page_lock_queues(); - for (i = 0; i < count; i++) + for (i = 0; i < count; i++) { + vm_page_lock(m[i]); + vm_page_lock_queues(); vm_page_free(m[i]); - vm_page_unlock_queues(); + vm_page_unlock_queues(); + vm_page_unlock(m[i]); + } vm_page_insert(page, object, offset); m[reqpage] = page; } Modified: head/sys/vm/sg_pager.c ============================================================================== --- head/sys/vm/sg_pager.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/vm/sg_pager.c Fri Apr 30 00:46:43 2010 (r207410) @@ -198,10 +198,13 @@ sg_pager_getpages(vm_object_t object, vm TAILQ_INSERT_TAIL(&object->un_pager.sgp.sgp_pglist, page, pageq); /* Free the original pages and insert this fake page into the object. */ - vm_page_lock_queues(); - for (i = 0; i < count; i++) + for (i = 0; i < count; i++) { + vm_page_lock(m[i]); + vm_page_lock_queues(); vm_page_free(m[i]); - vm_page_unlock_queues(); + vm_page_unlock_queues(); + vm_page_unlock(m[i]); + } vm_page_insert(page, object, offset); m[reqpage] = page; page->valid = VM_PAGE_BITS_ALL; Modified: head/sys/vm/swap_pager.c ============================================================================== --- head/sys/vm/swap_pager.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/vm/swap_pager.c Fri Apr 30 00:46:43 2010 (r207410) @@ -1137,12 +1137,21 @@ swap_pager_getpages(vm_object_t object, if (0 < i || j < count) { int k; - vm_page_lock_queues(); - for (k = 0; k < i; ++k) + + for (k = 0; k < i; ++k) { + vm_page_lock(m[k]); + vm_page_lock_queues(); swp_pager_free_nrpage(m[k]); - for (k = j; k < count; ++k) + vm_page_unlock_queues(); + vm_page_unlock(m[k]); + } + for (k = j; k < count; ++k) { + vm_page_lock(m[k]); + vm_page_lock_queues(); swp_pager_free_nrpage(m[k]); - vm_page_unlock_queues(); + vm_page_unlock_queues(); + vm_page_unlock(m[k]); + } } /* @@ -1497,7 +1506,7 @@ swp_pager_async_iodone(struct buf *bp) object = bp->b_pages[0]->object; VM_OBJECT_LOCK(object); } - vm_page_lock_queues(); + /* * cleanup pages. If an error occurs writing to swap, we are in * very serious trouble. If it happens to be a disk error, though, @@ -1509,6 +1518,8 @@ swp_pager_async_iodone(struct buf *bp) for (i = 0; i < bp->b_npages; ++i) { vm_page_t m = bp->b_pages[i]; + vm_page_lock(m); + vm_page_lock_queues(); m->oflags &= ~VPO_SWAPINPROG; if (bp->b_ioflags & BIO_ERROR) { @@ -1605,8 +1616,9 @@ swp_pager_async_iodone(struct buf *bp) if (vm_page_count_severe()) vm_page_try_to_cache(m); } + vm_page_unlock_queues(); + vm_page_unlock(m); } - vm_page_unlock_queues(); /* * adjust pip. NOTE: the original parent may still have its own @@ -1702,10 +1714,12 @@ swp_pager_force_pagein(vm_object_t objec m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL|VM_ALLOC_RETRY); if (m->valid == VM_PAGE_BITS_ALL) { vm_object_pip_subtract(object, 1); + vm_page_lock(m); vm_page_lock_queues(); vm_page_activate(m); vm_page_dirty(m); vm_page_unlock_queues(); + vm_page_unlock(m); vm_page_wakeup(m); vm_pager_page_unswapped(m); return; @@ -1714,10 +1728,12 @@ swp_pager_force_pagein(vm_object_t objec if (swap_pager_getpages(object, &m, 1, 0) != VM_PAGER_OK) panic("swap_pager_force_pagein: read from swap failed");/*XXX*/ vm_object_pip_subtract(object, 1); + vm_page_lock(m); vm_page_lock_queues(); vm_page_dirty(m); vm_page_dontneed(m); vm_page_unlock_queues(); + vm_page_unlock(m); vm_page_wakeup(m); vm_pager_page_unswapped(m); } Modified: head/sys/vm/uma_core.c ============================================================================== --- head/sys/vm/uma_core.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/vm/uma_core.c Fri Apr 30 00:46:43 2010 (r207410) @@ -1022,10 +1022,12 @@ obj_alloc(uma_zone_t zone, int bytes, u_ while (pages != startpages) { pages--; p = TAILQ_LAST(&object->memq, pglist); + vm_page_lock(p); vm_page_lock_queues(); vm_page_unwire(p, 0); vm_page_free(p); vm_page_unlock_queues(); + vm_page_unlock(p); } retkva = 0; goto done; Modified: head/sys/vm/vm_contig.c ============================================================================== --- head/sys/vm/vm_contig.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/vm/vm_contig.c Fri Apr 30 00:46:43 2010 (r207410) @@ -257,9 +257,11 @@ retry: i -= PAGE_SIZE; m = vm_page_lookup(object, OFF_TO_IDX(offset + i)); + vm_page_lock(m); vm_page_lock_queues(); vm_page_free(m); vm_page_unlock_queues(); + vm_page_unlock(m); } VM_OBJECT_UNLOCK(object); vm_map_delete(map, addr, addr + size); Modified: head/sys/vm/vm_fault.c ============================================================================== --- head/sys/vm/vm_fault.c Fri Apr 30 00:34:00 2010 (r207409) +++ head/sys/vm/vm_fault.c Fri Apr 30 00:46:43 2010 (r207410) @@ -137,9 +137,11 @@ release_page(struct faultstate *fs) { vm_page_wakeup(fs->m); + vm_page_lock(fs->m); vm_page_lock_queues(); vm_page_deactivate(fs->m); vm_page_unlock_queues(); + vm_page_unlock(fs->m); fs->m = NULL; } @@ -161,9 +163,11 @@ unlock_and_deallocate(struct faultstate VM_OBJECT_UNLOCK(fs->object); if (fs->object != fs->first_object) { VM_OBJECT_LOCK(fs->first_object); + vm_page_lock(fs->first_m); vm_page_lock_queues(); vm_page_free(fs->first_m); vm_page_unlock_queues(); + vm_page_unlock(fs->first_m); vm_object_pip_wakeup(fs->first_object); VM_OBJECT_UNLOCK(fs->first_object); fs->first_m = NULL; @@ -305,12 +309,14 @@ RetryFault:; * removes the page from the backing object, * which is not what we want. */ + vm_page_lock(fs.m); vm_page_lock_queues(); if ((fs.m->cow) && (fault_type & VM_PROT_WRITE) && (fs.object == fs.first_object)) { vm_page_cowfault(fs.m); vm_page_unlock_queues(); + vm_page_unlock(fs.m); unlock_and_deallocate(&fs); goto RetryFault; } @@ -333,12 +339,15 @@ RetryFault:; */ if ((fs.m->oflags & VPO_BUSY) || fs.m->busy) { vm_page_unlock_queues(); + vm_page_unlock(fs.m); VM_OBJECT_UNLOCK(fs.object); if (fs.object != fs.first_object) { VM_OBJECT_LOCK(fs.first_object); + vm_page_lock(fs.first_m); vm_page_lock_queues(); vm_page_free(fs.first_m); vm_page_unlock_queues(); + vm_page_unlock(fs.first_m); vm_object_pip_wakeup(fs.first_object); VM_OBJECT_UNLOCK(fs.first_object); fs.first_m = NULL; @@ -358,6 +367,7 @@ RetryFault:; } vm_pageq_remove(fs.m); vm_page_unlock_queues(); + vm_page_unlock(fs.m); /* * Mark page busy for other processes, and the @@ -481,17 +491,25 @@ readrest: continue; if (!are_queues_locked) { are_queues_locked = TRUE; + vm_page_lock(mt); + vm_page_lock_queues(); + } else { + vm_page_unlock_queues(); + vm_page_lock(mt); *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***