From owner-svn-src-head@FreeBSD.ORG Mon Mar 16 16:29:34 2015 Return-Path: Delivered-To: svn-src-head@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [8.8.178.115]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id D55644F4; Mon, 16 Mar 2015 16:29:34 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id B6587CBD; Mon, 16 Mar 2015 16:29:34 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.9/8.14.9) with ESMTP id t2GGTYrO001731; Mon, 16 Mar 2015 16:29:34 GMT (envelope-from nwhitehorn@FreeBSD.org) Received: (from nwhitehorn@localhost) by svn.freebsd.org (8.14.9/8.14.9/Submit) id t2GGTYQX001730; Mon, 16 Mar 2015 16:29:34 GMT (envelope-from nwhitehorn@FreeBSD.org) Message-Id: <201503161629.t2GGTYQX001730@svn.freebsd.org> X-Authentication-Warning: svn.freebsd.org: nwhitehorn set sender to nwhitehorn@FreeBSD.org using -f From: Nathan Whitehorn Date: Mon, 16 Mar 2015 16:29:34 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r280147 - head/sys/powerpc/pseries X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.18-1 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 16 Mar 2015 16:29:34 -0000 Author: nwhitehorn Date: Mon Mar 16 16:29:33 2015 New Revision: 280147 URL: https://svnweb.freebsd.org/changeset/base/280147 Log: Convert PTE eviction lock from an RW lock to a RM lock. It is held for writing approximately never (< 0.00000001% under heavy VM load, and it can go for months without ever being acquired in normal operation). This provides a 10% (2-minute) improvement in wall clock time for make -j32 buildworld on a 4-core 32-thread POWER8. Modified: head/sys/powerpc/pseries/mmu_phyp.c Modified: head/sys/powerpc/pseries/mmu_phyp.c ============================================================================== --- head/sys/powerpc/pseries/mmu_phyp.c Mon Mar 16 15:56:06 2015 (r280146) +++ head/sys/powerpc/pseries/mmu_phyp.c Mon Mar 16 16:29:33 2015 (r280147) @@ -30,7 +30,7 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include +#include #include #include #include @@ -59,7 +59,7 @@ __FBSDID("$FreeBSD$"); extern int n_slbs; -static struct rwlock mphyp_eviction_lock; +static struct rmlock mphyp_eviction_lock; /* * Kernel MMU interface @@ -119,7 +119,7 @@ mphyp_bootstrap(mmu_t mmup, vm_offset_t phandle_t dev, node, root; int idx, len, res; - rw_init(&mphyp_eviction_lock, "pte eviction"); + rm_init(&mphyp_eviction_lock, "pte eviction"); moea64_early_bootstrap(mmup, kernelstart, kernelend); @@ -262,6 +262,7 @@ mphyp_pte_synch(mmu_t mmu, struct pvo_en static int64_t mphyp_pte_clear(mmu_t mmu, struct pvo_entry *pvo, uint64_t ptebit) { + struct rm_priotracker track; int64_t refchg; uint64_t ptelo, junk; int err; @@ -274,11 +275,11 @@ mphyp_pte_clear(mmu_t mmu, struct pvo_en * shared eviction lock. */ PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); - rw_rlock(&mphyp_eviction_lock); + rm_rlock(&mphyp_eviction_lock, &track); refchg = mphyp_pte_synch(mmu, pvo); if (refchg < 0) { - rw_runlock(&mphyp_eviction_lock); + rm_runlock(&mphyp_eviction_lock, &track); return (refchg); } @@ -288,7 +289,7 @@ mphyp_pte_clear(mmu_t mmu, struct pvo_en * Pessimistically claim that, once modified, it stays so * forever and that it is never referenced. */ - rw_runlock(&mphyp_eviction_lock); + rm_runlock(&mphyp_eviction_lock, &track); return (refchg & ~LPTE_REF); } @@ -307,7 +308,7 @@ mphyp_pte_clear(mmu_t mmu, struct pvo_en refchg |= (ptelo & LPTE_REF); } - rw_runlock(&mphyp_eviction_lock); + rm_runlock(&mphyp_eviction_lock, &track); return (refchg); } @@ -376,6 +377,7 @@ mphyp_pte_spillable_ident(uintptr_t pteg static int mphyp_pte_insert(mmu_t mmu, struct pvo_entry *pvo) { + struct rm_priotracker track; int64_t result; struct lpte evicted, pte; uint64_t index, junk, lastptelo; @@ -387,7 +389,7 @@ mphyp_pte_insert(mmu_t mmu, struct pvo_e evicted.pte_hi = 0; /* Make sure further insertion is locked out during evictions */ - rw_rlock(&mphyp_eviction_lock); + rm_rlock(&mphyp_eviction_lock, &track); /* * First try primary hash. @@ -396,7 +398,7 @@ mphyp_pte_insert(mmu_t mmu, struct pvo_e result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, pte.pte_hi, pte.pte_lo, &index, &evicted.pte_lo, &junk); if (result == H_SUCCESS) { - rw_runlock(&mphyp_eviction_lock); + rm_runlock(&mphyp_eviction_lock, &track); pvo->pvo_pte.slot = index; return (0); } @@ -414,7 +416,7 @@ mphyp_pte_insert(mmu_t mmu, struct pvo_e result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, pte.pte_hi, pte.pte_lo, &index, &evicted.pte_lo, &junk); if (result == H_SUCCESS) { - rw_runlock(&mphyp_eviction_lock); + rm_runlock(&mphyp_eviction_lock, &track); pvo->pvo_pte.slot = index; return (0); } @@ -426,10 +428,8 @@ mphyp_pte_insert(mmu_t mmu, struct pvo_e */ /* Lock out all insertions for a bit */ - if (!rw_try_upgrade(&mphyp_eviction_lock)) { - rw_runlock(&mphyp_eviction_lock); - rw_wlock(&mphyp_eviction_lock); - } + rm_runlock(&mphyp_eviction_lock, &track); + rm_wlock(&mphyp_eviction_lock); index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted); if (index == -1L) { @@ -442,7 +442,7 @@ mphyp_pte_insert(mmu_t mmu, struct pvo_e if (index == -1L) { /* No freeable slots in either PTEG? We're hosed. */ - rw_wunlock(&mphyp_eviction_lock); + rm_wunlock(&mphyp_eviction_lock); panic("mphyp_pte_insert: overflow"); return (-1); } @@ -462,7 +462,7 @@ mphyp_pte_insert(mmu_t mmu, struct pvo_e */ result = phyp_pft_hcall(H_ENTER, H_EXACT, index, pte.pte_hi, pte.pte_lo, &index, &evicted.pte_lo, &junk); - rw_wunlock(&mphyp_eviction_lock); /* All clear */ + rm_wunlock(&mphyp_eviction_lock); /* All clear */ pvo->pvo_pte.slot = index; if (result == H_SUCCESS)