From owner-svn-src-all@freebsd.org Sat Dec 14 05:21:57 2019 Return-Path: Delivered-To: svn-src-all@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id AF84C1E20E0; Sat, 14 Dec 2019 05:21:57 +0000 (UTC) (envelope-from rlibby@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) server-signature RSA-PSS (4096 bits) client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 47ZbW941zwz3yY3; Sat, 14 Dec 2019 05:21:57 +0000 (UTC) (envelope-from rlibby@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 858981FE4C; Sat, 14 Dec 2019 05:21:57 +0000 (UTC) (envelope-from rlibby@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id xBE5Lv6K016859; Sat, 14 Dec 2019 05:21:57 GMT (envelope-from rlibby@FreeBSD.org) Received: (from rlibby@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id xBE5LvQm016857; Sat, 14 Dec 2019 05:21:57 GMT (envelope-from rlibby@FreeBSD.org) Message-Id: <201912140521.xBE5LvQm016857@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: rlibby set sender to rlibby@FreeBSD.org using -f From: Ryan Libby Date: Sat, 14 Dec 2019 05:21:57 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r355746 - head/sys/vm X-SVN-Group: head X-SVN-Commit-Author: rlibby X-SVN-Commit-Paths: head/sys/vm X-SVN-Commit-Revision: 355746 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 14 Dec 2019 05:21:57 -0000 Author: rlibby Date: Sat Dec 14 05:21:56 2019 New Revision: 355746 URL: https://svnweb.freebsd.org/changeset/base/355746 Log: uma dbg: flexible size for slab debug bitset too Recently (r355315) the size of the struct uma_slab bitset field us_free became dynamic instead of conservative. Now, make the debug bitset size dynamic too. The debug bitset is INVARIANTS-only, so in fact we don't care too much about the space savings that results from this, but enabling minimally-sized slabs on INVARIANTS builds is still important in order to be able to test new slab layouts effectively. Reviewed by: jeff (previous version), markj (previous version) Sponsored by: Dell EMC Isilon Differential Revision: https://reviews.freebsd.org/D22759 Modified: head/sys/vm/uma_core.c head/sys/vm/uma_int.h Modified: head/sys/vm/uma_core.c ============================================================================== --- head/sys/vm/uma_core.c Sat Dec 14 03:14:46 2019 (r355745) +++ head/sys/vm/uma_core.c Sat Dec 14 05:21:56 2019 (r355746) @@ -293,6 +293,8 @@ static int sysctl_handle_uma_zone_flags(SYSCTL_HANDLER static int sysctl_handle_uma_slab_efficiency(SYSCTL_HANDLER_ARGS); #ifdef INVARIANTS +static inline struct noslabbits *slab_dbg_bits(uma_slab_t slab, uma_keg_t keg); + static bool uma_dbg_kskip(uma_keg_t keg, void *mem); static bool uma_dbg_zskip(uma_zone_t zone, void *mem); static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item); @@ -1204,7 +1206,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int dom slab->us_domain = domain; BIT_FILL(keg->uk_ipers, &slab->us_free); #ifdef INVARIANTS - BIT_ZERO(SLAB_MAX_SETSIZE, &slab->us_debugfree); + BIT_ZERO(keg->uk_ipers, slab_dbg_bits(slab, keg)); #endif if (keg->uk_init != NULL) { @@ -1487,6 +1489,15 @@ zero_init(void *mem, int size, int flags) return (0); } +#ifdef INVARIANTS +struct noslabbits * +slab_dbg_bits(uma_slab_t slab, uma_keg_t keg) +{ + + return ((void *)((char *)&slab->us_free + BITSET_SIZE(keg->uk_ipers))); +} +#endif + /* * Actual size of embedded struct slab (!OFFPAGE). */ @@ -1495,7 +1506,7 @@ slab_sizeof(int nitems) { size_t s; - s = sizeof(struct uma_slab) + BITSET_SIZE(nitems); + s = sizeof(struct uma_slab) + BITSET_SIZE(nitems) * SLAB_BITSETS; return (roundup(s, UMA_ALIGN_PTR + 1)); } @@ -4541,12 +4552,10 @@ uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void * keg = zone->uz_keg; freei = slab_item_index(slab, keg, item); - if (BIT_ISSET(SLAB_MAX_SETSIZE, freei, &slab->us_debugfree)) + if (BIT_ISSET(keg->uk_ipers, freei, slab_dbg_bits(slab, keg))) panic("Duplicate alloc of %p from zone %p(%s) slab %p(%d)\n", item, zone, zone->uz_name, slab, freei); - BIT_SET_ATOMIC(SLAB_MAX_SETSIZE, freei, &slab->us_debugfree); - - return; + BIT_SET_ATOMIC(keg->uk_ipers, freei, slab_dbg_bits(slab, keg)); } /* @@ -4577,11 +4586,11 @@ uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *i panic("Unaligned free of %p from zone %p(%s) slab %p(%d)\n", item, zone, zone->uz_name, slab, freei); - if (!BIT_ISSET(SLAB_MAX_SETSIZE, freei, &slab->us_debugfree)) + if (!BIT_ISSET(keg->uk_ipers, freei, slab_dbg_bits(slab, keg))) panic("Duplicate free of %p from zone %p(%s) slab %p(%d)\n", item, zone, zone->uz_name, slab, freei); - BIT_CLR_ATOMIC(SLAB_MAX_SETSIZE, freei, &slab->us_debugfree); + BIT_CLR_ATOMIC(keg->uk_ipers, freei, slab_dbg_bits(slab, keg)); } #endif /* INVARIANTS */ Modified: head/sys/vm/uma_int.h ============================================================================== --- head/sys/vm/uma_int.h Sat Dec 14 03:14:46 2019 (r355745) +++ head/sys/vm/uma_int.h Sat Dec 14 05:21:56 2019 (r355746) @@ -254,6 +254,7 @@ struct uma_keg { }; typedef struct uma_keg * uma_keg_t; +#ifdef _KERNEL /* * Free bits per-slab. */ @@ -271,17 +272,26 @@ struct uma_slab { uint16_t us_freecount; /* How many are free? */ uint8_t us_flags; /* Page flags see uma.h */ uint8_t us_domain; /* Backing NUMA domain. */ -#ifdef INVARIANTS - struct slabbits us_debugfree; /* Debug bitmask. */ -#endif - struct noslabbits us_free; /* Free bitmask. */ + struct noslabbits us_free; /* Free bitmask, flexible. */ }; +_Static_assert(sizeof(struct uma_slab) == offsetof(struct uma_slab, us_free), + "us_free field must be last"); #if MAXMEMDOM >= 255 #error "Slab domain type insufficient" #endif typedef struct uma_slab * uma_slab_t; +/* + * On INVARIANTS builds, the slab contains a second bitset of the same size, + * "dbg_bits", which is laid out immediately after us_free. + */ +#ifdef INVARIANTS +#define SLAB_BITSETS 2 +#else +#define SLAB_BITSETS 1 +#endif + /* These three functions are for embedded (!OFFPAGE) use only. */ size_t slab_sizeof(int nitems); size_t slab_space(int nitems); @@ -293,7 +303,10 @@ int slab_ipers(size_t size, int align); */ struct uma_hash_slab { struct uma_slab uhs_slab; /* Must be first. */ - struct slabbits uhs_bits; /* Must be second. */ + struct slabbits uhs_bits1; /* Must be second. */ +#ifdef INVARIANTS + struct slabbits uhs_bits2; /* Must be third. */ +#endif LIST_ENTRY(uma_hash_slab) uhs_hlink; /* Link for hash table */ uint8_t *uhs_data; /* First item */ }; @@ -327,6 +340,7 @@ slab_item_index(uma_slab_t slab, uma_keg_t keg, void * data = (uintptr_t)slab_data(slab, keg); return (((uintptr_t)item - data) / keg->uk_rsize); } +#endif /* _KERNEL */ TAILQ_HEAD(uma_bucketlist, uma_bucket);