Skip site navigation (1)Skip section navigation (2)
Date:      Tue, 15 Jan 2019 18:24:34 +0000 (UTC)
From:      Gleb Smirnoff <glebius@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r343051 - in head/sys: compat/linuxkpi/common/src vm
Message-ID:  <201901151824.x0FIOYwP025806@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: glebius
Date: Tue Jan 15 18:24:34 2019
New Revision: 343051
URL: https://svnweb.freebsd.org/changeset/base/343051

Log:
  Make uz_allocs, uz_frees and uz_fails counter(9). This removes some
  atomic updates and reduces amount of data protected by zone lock.
  
  During startup point these fields to EARLY_COUNTER. After startup
  allocate them for all early zones.
  
  Tested by:	pho

Modified:
  head/sys/compat/linuxkpi/common/src/linux_page.c
  head/sys/vm/uma_core.c
  head/sys/vm/uma_dbg.c
  head/sys/vm/uma_int.h

Modified: head/sys/compat/linuxkpi/common/src/linux_page.c
==============================================================================
--- head/sys/compat/linuxkpi/common/src/linux_page.c	Tue Jan 15 18:22:16 2019	(r343050)
+++ head/sys/compat/linuxkpi/common/src/linux_page.c	Tue Jan 15 18:24:34 2019	(r343051)
@@ -31,6 +31,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/counter.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
 #include <sys/sysctl.h>

Modified: head/sys/vm/uma_core.c
==============================================================================
--- head/sys/vm/uma_core.c	Tue Jan 15 18:22:16 2019	(r343050)
+++ head/sys/vm/uma_core.c	Tue Jan 15 18:24:34 2019	(r343051)
@@ -1715,6 +1715,15 @@ keg_ctor(void *mem, int size, void *udata, int flags)
 	return (0);
 }
 
+static void
+zone_alloc_counters(uma_zone_t zone)
+{
+
+	zone->uz_allocs = counter_u64_alloc(M_WAITOK);
+	zone->uz_frees = counter_u64_alloc(M_WAITOK);
+	zone->uz_fails = counter_u64_alloc(M_WAITOK);
+}
+
 /*
  * Zone header ctor.  This initializes all fields, locks, etc.
  *
@@ -1736,9 +1745,6 @@ zone_ctor(void *mem, int size, void *udata, int flags)
 	zone->uz_slab = zone_fetch_slab;
 	zone->uz_init = NULL;
 	zone->uz_fini = NULL;
-	zone->uz_allocs = 0;
-	zone->uz_frees = 0;
-	zone->uz_fails = 0;
 	zone->uz_sleeps = 0;
 	zone->uz_count = 0;
 	zone->uz_count_min = 0;
@@ -1750,6 +1756,14 @@ zone_ctor(void *mem, int size, void *udata, int flags)
 	zone->uz_bkt_max = ULONG_MAX;
 	timevalclear(&zone->uz_ratecheck);
 
+	if (__predict_true(booted == BOOT_RUNNING))
+		zone_alloc_counters(zone);
+	else {
+		zone->uz_allocs = EARLY_COUNTER;
+		zone->uz_frees = EARLY_COUNTER;
+		zone->uz_fails = EARLY_COUNTER;
+	}
+
 	/*
 	 * This is a pure cache zone, no kegs.
 	 */
@@ -1908,6 +1922,9 @@ zone_dtor(void *arg, int size, void *udata)
 		rw_wunlock(&uma_rwlock);
 		zone_free_item(kegs, keg, NULL, SKIP_NONE);
 	}
+	counter_u64_free(zone->uz_allocs);
+	counter_u64_free(zone->uz_frees);
+	counter_u64_free(zone->uz_fails);
 	if (zone->uz_lockptr == &zone->uz_lock)
 		ZONE_LOCK_FINI(zone);
 }
@@ -1928,12 +1945,19 @@ zone_foreach(void (*zfunc)(uma_zone_t))
 	uma_keg_t keg;
 	uma_zone_t zone;
 
-	rw_rlock(&uma_rwlock);
+	/*
+	 * Before BOOT_RUNNING we are guaranteed to be single
+	 * threaded, so locking isn't needed. Startup functions
+	 * are allowed to use M_WAITOK.
+	 */
+	if (__predict_true(booted == BOOT_RUNNING))
+		rw_rlock(&uma_rwlock);
 	LIST_FOREACH(keg, &uma_kegs, uk_link) {
 		LIST_FOREACH(zone, &keg->uk_zones, uz_link)
 			zfunc(zone);
 	}
-	rw_runlock(&uma_rwlock);
+	if (__predict_true(booted == BOOT_RUNNING))
+		rw_runlock(&uma_rwlock);
 }
 
 /*
@@ -2109,6 +2133,7 @@ uma_startup3(void)
 	uma_dbg_cnt = counter_u64_alloc(M_WAITOK);
 	uma_skip_cnt = counter_u64_alloc(M_WAITOK);
 #endif
+	zone_foreach(zone_alloc_counters);
 	callout_init(&uma_callout, 1);
 	callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
 	booted = BOOT_RUNNING;
@@ -2387,7 +2412,7 @@ zalloc_start:
 		    zone->uz_dtor != trash_dtor) &&
 #endif
 		    zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
-			atomic_add_long(&zone->uz_fails, 1);
+			counter_u64_add(zone->uz_fails, 1);
 			zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT);
 			return (NULL);
 		}
@@ -2845,7 +2870,7 @@ zone_alloc_bucket(uma_zone_t zone, void *udata, int do
 
 	if (bucket->ub_cnt == 0) {
 		bucket_free(zone, bucket, udata);
-		atomic_add_long(&zone->uz_fails, 1);
+		counter_u64_add(zone->uz_fails, 1);
 		return (NULL);
 	}
 
@@ -2905,7 +2930,6 @@ zone_alloc_item_locked(uma_zone_t zone, void *udata, i
 	}
 
 	zone->uz_items++;
-	zone->uz_allocs++;
 	ZONE_UNLOCK(zone);
 
 	if (domain != UMA_ANYDOMAIN) {
@@ -2947,6 +2971,7 @@ zone_alloc_item_locked(uma_zone_t zone, void *udata, i
 	if (flags & M_ZERO)
 		uma_zero_item(item, zone);
 
+	counter_u64_add(zone->uz_allocs, 1);
 	CTR3(KTR_UMA, "zone_alloc_item item %p from %s(%p)", item,
 	    zone->uz_name, zone);
 
@@ -2955,9 +2980,8 @@ zone_alloc_item_locked(uma_zone_t zone, void *udata, i
 fail:
 	ZONE_LOCK(zone);
 	zone->uz_items--;
-	zone->uz_allocs--;
 	ZONE_UNLOCK(zone);
-	atomic_add_long(&zone->uz_fails, 1);
+	counter_u64_add(zone->uz_fails, 1);
 	CTR2(KTR_UMA, "zone_alloc_item failed from %s(%p)",
 	    zone->uz_name, zone);
 	return (NULL);
@@ -3268,8 +3292,9 @@ zone_free_item(uma_zone_t zone, void *item, void *udat
 	if (skip & SKIP_CNT)
 		return;
 
+	counter_u64_add(zone->uz_frees, 1);
+
 	ZONE_LOCK(zone);
-	zone->uz_frees++;
 	zone->uz_items--;
 	if (zone->uz_sleepers > 0 && zone->uz_items < zone->uz_max_items)
 		wakeup_one(zone);
@@ -3358,7 +3383,8 @@ uma_zone_get_cur(uma_zone_t zone)
 	u_int i;
 
 	ZONE_LOCK(zone);
-	nitems = zone->uz_allocs - zone->uz_frees;
+	nitems = counter_u64_fetch(zone->uz_allocs) -
+	    counter_u64_fetch(zone->uz_frees);
 	CPU_FOREACH(i) {
 		/*
 		 * See the comment in sysctl_vm_zone_stats() regarding the
@@ -3801,8 +3827,8 @@ uma_zone_sumstat(uma_zone_t z, long *cachefreep, uint6
 		allocs += cache->uc_allocs;
 		frees += cache->uc_frees;
 	}
-	allocs += z->uz_allocs;
-	frees += z->uz_frees;
+	allocs += counter_u64_fetch(z->uz_allocs);
+	frees += counter_u64_fetch(z->uz_frees);
 	sleeps += z->uz_sleeps;
 	if (cachefreep != NULL)
 		*cachefreep = cachefree;
@@ -3895,9 +3921,9 @@ sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS)
 				zdom = &z->uz_domain[i];
 				uth.uth_zone_free += zdom->uzd_nitems;
 			}
-			uth.uth_allocs = z->uz_allocs;
-			uth.uth_frees = z->uz_frees;
-			uth.uth_fails = z->uz_fails;
+			uth.uth_allocs = counter_u64_fetch(z->uz_allocs);
+			uth.uth_frees = counter_u64_fetch(z->uz_frees);
+			uth.uth_fails = counter_u64_fetch(z->uz_fails);
 			uth.uth_sleeps = z->uz_sleeps;
 			/*
 			 * While it is not normally safe to access the cache
@@ -4105,8 +4131,8 @@ DB_SHOW_COMMAND(uma, db_show_uma)
 	LIST_FOREACH(kz, &uma_kegs, uk_link) {
 		LIST_FOREACH(z, &kz->uk_zones, uz_link) {
 			if (kz->uk_flags & UMA_ZFLAG_INTERNAL) {
-				allocs = z->uz_allocs;
-				frees = z->uz_frees;
+				allocs = counter_u64_fetch(z->uz_allocs);
+				frees = counter_u64_fetch(z->uz_frees);
 				sleeps = z->uz_sleeps;
 				cachefree = 0;
 			} else

Modified: head/sys/vm/uma_dbg.c
==============================================================================
--- head/sys/vm/uma_dbg.c	Tue Jan 15 18:22:16 2019	(r343050)
+++ head/sys/vm/uma_dbg.c	Tue Jan 15 18:24:34 2019	(r343051)
@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bitset.h>
+#include <sys/counter.h>
 #include <sys/kernel.h>
 #include <sys/types.h>
 #include <sys/queue.h>

Modified: head/sys/vm/uma_int.h
==============================================================================
--- head/sys/vm/uma_int.h	Tue Jan 15 18:22:16 2019	(r343050)
+++ head/sys/vm/uma_int.h	Tue Jan 15 18:24:34 2019	(r343051)
@@ -364,10 +364,10 @@ struct uma_zone {
 	uint16_t	uz_count_min;	/* Minimal amount of items in bucket */
 
 	/* Offset 256, stats. */
-	uint64_t	uz_allocs UMA_ALIGN; /* Total number of allocations */
+	counter_u64_t	uz_allocs; /* Total number of allocations */
+	counter_u64_t	uz_frees;	/* Total number of frees */
+	counter_u64_t	uz_fails;	/* Total number of alloc failures */
 	uint64_t	uz_sleeps;	/* Total number of alloc sleeps */
-	uint64_t	uz_frees;	/* Total number of frees */
-	volatile u_long	uz_fails;	/* Total number of alloc failures */
 
 	/*
 	 * This HAS to be the last item because we adjust the zone size



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201901151824.x0FIOYwP025806>