Date: Tue, 15 Jan 2019 18:24:34 +0000 (UTC) From: Gleb Smirnoff <glebius@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r343051 - in head/sys: compat/linuxkpi/common/src vm Message-ID: <201901151824.x0FIOYwP025806@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: glebius Date: Tue Jan 15 18:24:34 2019 New Revision: 343051 URL: https://svnweb.freebsd.org/changeset/base/343051 Log: Make uz_allocs, uz_frees and uz_fails counter(9). This removes some atomic updates and reduces amount of data protected by zone lock. During startup point these fields to EARLY_COUNTER. After startup allocate them for all early zones. Tested by: pho Modified: head/sys/compat/linuxkpi/common/src/linux_page.c head/sys/vm/uma_core.c head/sys/vm/uma_dbg.c head/sys/vm/uma_int.h Modified: head/sys/compat/linuxkpi/common/src/linux_page.c ============================================================================== --- head/sys/compat/linuxkpi/common/src/linux_page.c Tue Jan 15 18:22:16 2019 (r343050) +++ head/sys/compat/linuxkpi/common/src/linux_page.c Tue Jan 15 18:24:34 2019 (r343051) @@ -31,6 +31,7 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/systm.h> +#include <sys/counter.h> #include <sys/malloc.h> #include <sys/kernel.h> #include <sys/sysctl.h> Modified: head/sys/vm/uma_core.c ============================================================================== --- head/sys/vm/uma_core.c Tue Jan 15 18:22:16 2019 (r343050) +++ head/sys/vm/uma_core.c Tue Jan 15 18:24:34 2019 (r343051) @@ -1715,6 +1715,15 @@ keg_ctor(void *mem, int size, void *udata, int flags) return (0); } +static void +zone_alloc_counters(uma_zone_t zone) +{ + + zone->uz_allocs = counter_u64_alloc(M_WAITOK); + zone->uz_frees = counter_u64_alloc(M_WAITOK); + zone->uz_fails = counter_u64_alloc(M_WAITOK); +} + /* * Zone header ctor. This initializes all fields, locks, etc. * @@ -1736,9 +1745,6 @@ zone_ctor(void *mem, int size, void *udata, int flags) zone->uz_slab = zone_fetch_slab; zone->uz_init = NULL; zone->uz_fini = NULL; - zone->uz_allocs = 0; - zone->uz_frees = 0; - zone->uz_fails = 0; zone->uz_sleeps = 0; zone->uz_count = 0; zone->uz_count_min = 0; @@ -1750,6 +1756,14 @@ zone_ctor(void *mem, int size, void *udata, int flags) zone->uz_bkt_max = ULONG_MAX; timevalclear(&zone->uz_ratecheck); + if (__predict_true(booted == BOOT_RUNNING)) + zone_alloc_counters(zone); + else { + zone->uz_allocs = EARLY_COUNTER; + zone->uz_frees = EARLY_COUNTER; + zone->uz_fails = EARLY_COUNTER; + } + /* * This is a pure cache zone, no kegs. */ @@ -1908,6 +1922,9 @@ zone_dtor(void *arg, int size, void *udata) rw_wunlock(&uma_rwlock); zone_free_item(kegs, keg, NULL, SKIP_NONE); } + counter_u64_free(zone->uz_allocs); + counter_u64_free(zone->uz_frees); + counter_u64_free(zone->uz_fails); if (zone->uz_lockptr == &zone->uz_lock) ZONE_LOCK_FINI(zone); } @@ -1928,12 +1945,19 @@ zone_foreach(void (*zfunc)(uma_zone_t)) uma_keg_t keg; uma_zone_t zone; - rw_rlock(&uma_rwlock); + /* + * Before BOOT_RUNNING we are guaranteed to be single + * threaded, so locking isn't needed. Startup functions + * are allowed to use M_WAITOK. + */ + if (__predict_true(booted == BOOT_RUNNING)) + rw_rlock(&uma_rwlock); LIST_FOREACH(keg, &uma_kegs, uk_link) { LIST_FOREACH(zone, &keg->uk_zones, uz_link) zfunc(zone); } - rw_runlock(&uma_rwlock); + if (__predict_true(booted == BOOT_RUNNING)) + rw_runlock(&uma_rwlock); } /* @@ -2109,6 +2133,7 @@ uma_startup3(void) uma_dbg_cnt = counter_u64_alloc(M_WAITOK); uma_skip_cnt = counter_u64_alloc(M_WAITOK); #endif + zone_foreach(zone_alloc_counters); callout_init(&uma_callout, 1); callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); booted = BOOT_RUNNING; @@ -2387,7 +2412,7 @@ zalloc_start: zone->uz_dtor != trash_dtor) && #endif zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) { - atomic_add_long(&zone->uz_fails, 1); + counter_u64_add(zone->uz_fails, 1); zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT); return (NULL); } @@ -2845,7 +2870,7 @@ zone_alloc_bucket(uma_zone_t zone, void *udata, int do if (bucket->ub_cnt == 0) { bucket_free(zone, bucket, udata); - atomic_add_long(&zone->uz_fails, 1); + counter_u64_add(zone->uz_fails, 1); return (NULL); } @@ -2905,7 +2930,6 @@ zone_alloc_item_locked(uma_zone_t zone, void *udata, i } zone->uz_items++; - zone->uz_allocs++; ZONE_UNLOCK(zone); if (domain != UMA_ANYDOMAIN) { @@ -2947,6 +2971,7 @@ zone_alloc_item_locked(uma_zone_t zone, void *udata, i if (flags & M_ZERO) uma_zero_item(item, zone); + counter_u64_add(zone->uz_allocs, 1); CTR3(KTR_UMA, "zone_alloc_item item %p from %s(%p)", item, zone->uz_name, zone); @@ -2955,9 +2980,8 @@ zone_alloc_item_locked(uma_zone_t zone, void *udata, i fail: ZONE_LOCK(zone); zone->uz_items--; - zone->uz_allocs--; ZONE_UNLOCK(zone); - atomic_add_long(&zone->uz_fails, 1); + counter_u64_add(zone->uz_fails, 1); CTR2(KTR_UMA, "zone_alloc_item failed from %s(%p)", zone->uz_name, zone); return (NULL); @@ -3268,8 +3292,9 @@ zone_free_item(uma_zone_t zone, void *item, void *udat if (skip & SKIP_CNT) return; + counter_u64_add(zone->uz_frees, 1); + ZONE_LOCK(zone); - zone->uz_frees++; zone->uz_items--; if (zone->uz_sleepers > 0 && zone->uz_items < zone->uz_max_items) wakeup_one(zone); @@ -3358,7 +3383,8 @@ uma_zone_get_cur(uma_zone_t zone) u_int i; ZONE_LOCK(zone); - nitems = zone->uz_allocs - zone->uz_frees; + nitems = counter_u64_fetch(zone->uz_allocs) - + counter_u64_fetch(zone->uz_frees); CPU_FOREACH(i) { /* * See the comment in sysctl_vm_zone_stats() regarding the @@ -3801,8 +3827,8 @@ uma_zone_sumstat(uma_zone_t z, long *cachefreep, uint6 allocs += cache->uc_allocs; frees += cache->uc_frees; } - allocs += z->uz_allocs; - frees += z->uz_frees; + allocs += counter_u64_fetch(z->uz_allocs); + frees += counter_u64_fetch(z->uz_frees); sleeps += z->uz_sleeps; if (cachefreep != NULL) *cachefreep = cachefree; @@ -3895,9 +3921,9 @@ sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) zdom = &z->uz_domain[i]; uth.uth_zone_free += zdom->uzd_nitems; } - uth.uth_allocs = z->uz_allocs; - uth.uth_frees = z->uz_frees; - uth.uth_fails = z->uz_fails; + uth.uth_allocs = counter_u64_fetch(z->uz_allocs); + uth.uth_frees = counter_u64_fetch(z->uz_frees); + uth.uth_fails = counter_u64_fetch(z->uz_fails); uth.uth_sleeps = z->uz_sleeps; /* * While it is not normally safe to access the cache @@ -4105,8 +4131,8 @@ DB_SHOW_COMMAND(uma, db_show_uma) LIST_FOREACH(kz, &uma_kegs, uk_link) { LIST_FOREACH(z, &kz->uk_zones, uz_link) { if (kz->uk_flags & UMA_ZFLAG_INTERNAL) { - allocs = z->uz_allocs; - frees = z->uz_frees; + allocs = counter_u64_fetch(z->uz_allocs); + frees = counter_u64_fetch(z->uz_frees); sleeps = z->uz_sleeps; cachefree = 0; } else Modified: head/sys/vm/uma_dbg.c ============================================================================== --- head/sys/vm/uma_dbg.c Tue Jan 15 18:22:16 2019 (r343050) +++ head/sys/vm/uma_dbg.c Tue Jan 15 18:24:34 2019 (r343051) @@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/systm.h> #include <sys/bitset.h> +#include <sys/counter.h> #include <sys/kernel.h> #include <sys/types.h> #include <sys/queue.h> Modified: head/sys/vm/uma_int.h ============================================================================== --- head/sys/vm/uma_int.h Tue Jan 15 18:22:16 2019 (r343050) +++ head/sys/vm/uma_int.h Tue Jan 15 18:24:34 2019 (r343051) @@ -364,10 +364,10 @@ struct uma_zone { uint16_t uz_count_min; /* Minimal amount of items in bucket */ /* Offset 256, stats. */ - uint64_t uz_allocs UMA_ALIGN; /* Total number of allocations */ + counter_u64_t uz_allocs; /* Total number of allocations */ + counter_u64_t uz_frees; /* Total number of frees */ + counter_u64_t uz_fails; /* Total number of alloc failures */ uint64_t uz_sleeps; /* Total number of alloc sleeps */ - uint64_t uz_frees; /* Total number of frees */ - volatile u_long uz_fails; /* Total number of alloc failures */ /* * This HAS to be the last item because we adjust the zone size
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201901151824.x0FIOYwP025806>