Date: Fri, 2 Apr 2010 21:15:42 +0000 (UTC) From: Kip Macy <kmacy@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r206114 - user/kmacy/head_page_lock/sys/amd64/amd64 Message-ID: <201004022115.o32LFg3x078828@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kmacy Date: Fri Apr 2 21:15:42 2010 New Revision: 206114 URL: http://svn.freebsd.org/changeset/base/206114 Log: - reduce coverage of pv_lock - make pv_entry_count atomic to eliminate pv_lock from the exit path Modified: user/kmacy/head_page_lock/sys/amd64/amd64/pmap.c Modified: user/kmacy/head_page_lock/sys/amd64/amd64/pmap.c ============================================================================== --- user/kmacy/head_page_lock/sys/amd64/amd64/pmap.c Fri Apr 2 21:05:28 2010 (r206113) +++ user/kmacy/head_page_lock/sys/amd64/amd64/pmap.c Fri Apr 2 21:15:42 2010 (r206114) @@ -160,15 +160,13 @@ __FBSDID("$FreeBSD$"); #define PMAP_INLINE #endif -#define PV_STATS #ifdef PV_STATS #define PV_STAT(x) do { x ; } while (0) #else #define PV_STAT(x) do { } while (0) #endif -#define CACHE_LINE_FETCH_SIZE 128 -#define PA_LOCK_PAD CACHE_LINE_FETCH_SIZE +#define PA_LOCK_PAD CACHE_LINE_SIZE struct vp_lock { struct mtx vp_lock; @@ -184,10 +182,10 @@ struct vp_lock { #define PA_UNLOCK(pa) mtx_unlock(PA_LOCKPTR(pa)) #define PA_LOCK_ASSERT(pa, a) mtx_assert(PA_LOCKPTR(pa), (a)) -#define PA_LOCK_COUNT 64 +#define PA_LOCK_COUNT 256 -struct mtx pv_lock __aligned(128); -struct vp_lock pa_lock[PA_LOCK_COUNT] __aligned(128); +struct mtx pv_lock __aligned(CACHE_LINE_SIZE); +struct vp_lock pa_lock[PA_LOCK_COUNT] __aligned(CACHE_LINE_SIZE); struct pmap kernel_pmap_store; @@ -228,7 +226,8 @@ static u_int64_t DMPDPphys; /* phys addr /* * Data for the pv entry allocation mechanism */ -static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0; +static int pv_entry_count __aligned(CACHE_LINE_SIZE); +static int pv_entry_max = 0, pv_entry_high_water = 0; static struct md_page *pv_table; static int shpgperproc = PMAP_SHPGPERPROC; @@ -313,18 +312,24 @@ ls_init(struct lock_stack *ls) ls->ls_top = 0; } + +#define ls_push(ls, m) _ls_push((ls), (m), LOCK_FILE, LOCK_LINE) + static void -ls_push(struct lock_stack *ls, struct mtx *lock) +_ls_push(struct lock_stack *ls, struct mtx *lock, char *file, int line) { KASSERT(ls->ls_top < LS_MAX, ("lock stack overflow")); ls->ls_array[ls->ls_top] = lock; ls->ls_top++; - mtx_lock(lock); +#if LOCK_DEBUG > 0 || defined(MUTEX_NOINLINE) + _mtx_lock_flags(lock, 0, file, line); +#else + _get_sleep_lock(lock, curthread, 0, file, line); +#endif } - static int ls_trypush(struct lock_stack *ls, struct mtx *lock) { @@ -542,9 +547,9 @@ pa_tryrelock(pmap_t pmap, vm_paddr_t pa, if (PA_TRYLOCK(pa)) return 0; PMAP_UNLOCK(pmap); + atomic_add_int((volatile int *)&pmap_tryrelock_restart, 1); PA_LOCK(pa); PMAP_LOCK(pmap); - atomic_add_int((volatile int *)&pmap_tryrelock_restart, 1); return (EAGAIN); } @@ -2122,10 +2127,11 @@ free_pv_entry(pmap_t pmap, pv_entry_t pv int idx, field, bit; PMAP_LOCK_ASSERT(pmap, MA_OWNED); - mtx_lock(&pv_lock); + + atomic_add_int(&pv_entry_count, -1); PV_STAT(pv_entry_frees++); PV_STAT(pv_entry_spare++); - pv_entry_count--; + mtx_lock(&pv_lock); pc = pv_to_chunk(pv); idx = pv - &pc->pc_pventry[0]; field = idx / 64; @@ -2169,9 +2175,10 @@ get_pv_entry(pmap_t pmap) vm_page_t m; PMAP_LOCK_ASSERT(pmap, MA_OWNED); - mtx_lock(&pv_lock); + + atomic_add_int(&pv_entry_count, 1); PV_STAT(pv_entry_allocs++); - pv_entry_count++; + mtx_lock(&pv_lock); if (pv_entry_count > pv_entry_high_water) if (ratecheck(&lastprint, &printinterval)) printf("Approaching the limit on PV entries, consider " @@ -2195,8 +2202,8 @@ get_pv_entry(pmap_t pmap) TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); } - PV_STAT(pv_entry_spare--); mtx_unlock(&pv_lock); + PV_STAT(pv_entry_spare--); return (pv); } } @@ -2205,9 +2212,9 @@ get_pv_entry(pmap_t pmap) VM_ALLOC_SYSTEM : VM_ALLOC_NORMAL) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); if (m == NULL) { - pv_entry_count--; - PV_STAT(pc_chunk_tryfail++); mtx_unlock(&pv_lock); + PV_STAT(pc_chunk_tryfail++); + atomic_add_int(&pv_entry_count, -1); return (NULL); } PV_STAT(pc_chunk_count++); @@ -4366,12 +4373,10 @@ restart: } else vm_page_dirty(m); } - mtx_lock(&pv_lock); /* Mark free */ PV_STAT(pv_entry_frees++); PV_STAT(pv_entry_spare++); - pv_entry_count--; - mtx_unlock(&pv_lock); + atomic_add_int(&pv_entry_count, -1); pc->pc_map[field] |= bitmask; if ((tpte & PG_PS) != 0) { pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201004022115.o32LFg3x078828>