Date: Sun, 6 Dec 2020 22:45:51 +0000 (UTC) From: Mark Johnston <markj@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r368400 - head/sys/vm Message-ID: <202012062245.0B6MjpvX032617@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: markj Date: Sun Dec 6 22:45:50 2020 New Revision: 368400 URL: https://svnweb.freebsd.org/changeset/base/368400 Log: uma: Make uma_zone_set_maxcache() work better with small limits The old implementation chose the largest bucket zone such that if the per-CPU caches are fully populated, the total number of items cached is no larger than the specified limit. If no such zone existed, UMA would not do any caching. We can now use uz_bucket_size_max to set a precise limit on the number of items in a zone's bucket, so the total size of per-CPU caches can be bounded more easily. Implement a new policy in uma_zone_set_maxcache(): choose a bucket size such that up to half of the limit can be cached in per-CPU caches, with the rest going to the full bucket cache. This fixes a problem with the kstack_cache zone: the limit of 4 * mp_ncpus items meant that the zone would not do any caching, defeating the whole purpose of the zone. That's because the smallest bucket size holds up to 2 items and we may cache up to 3 full buckets per CPU, and 2 * 3 * mp_ncpus > 4 * mp_ncpus. Reported by: mjg Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D27168 Modified: head/sys/vm/uma.h head/sys/vm/uma_core.c Modified: head/sys/vm/uma.h ============================================================================== --- head/sys/vm/uma.h Sun Dec 6 22:45:39 2020 (r368399) +++ head/sys/vm/uma.h Sun Dec 6 22:45:50 2020 (r368400) @@ -492,7 +492,7 @@ void uma_zone_reserve(uma_zone_t zone, int nitems); int uma_zone_reserve_kva(uma_zone_t zone, int nitems); /* - * Sets a high limit on the number of items allowed in a zone + * Sets an upper limit on the number of items allocated from a zone * * Arguments: * zone The zone to limit @@ -504,7 +504,7 @@ int uma_zone_reserve_kva(uma_zone_t zone, int nitems); int uma_zone_set_max(uma_zone_t zone, int nitems); /* - * Sets a high limit on the number of items allowed in zone's bucket cache + * Sets an upper limit on the number of items allowed in zone's caches * * Arguments: * zone The zone to limit Modified: head/sys/vm/uma_core.c ============================================================================== --- head/sys/vm/uma_core.c Sun Dec 6 22:45:39 2020 (r368399) +++ head/sys/vm/uma_core.c Sun Dec 6 22:45:50 2020 (r368400) @@ -438,27 +438,6 @@ bucket_zone_lookup(int entries) return (ubz); } -static struct uma_bucket_zone * -bucket_zone_max(uma_zone_t zone, int nitems) -{ - struct uma_bucket_zone *ubz; - int bpcpu; - - bpcpu = 2; - if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) - /* Count the cross-domain bucket. */ - bpcpu++; - - for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) - if (ubz->ubz_entries * bpcpu * mp_ncpus > nitems) - break; - if (ubz == &bucket_zones[0]) - ubz = NULL; - else - ubz--; - return (ubz); -} - static int bucket_select(int size) { @@ -2478,10 +2457,10 @@ zone_alloc_sysctl(uma_zone_t zone, void *unused) SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, "items", CTLFLAG_RD | CTLTYPE_U64 | CTLFLAG_MPSAFE, zone, 0, sysctl_handle_uma_zone_items, "QU", - "current number of allocated items if limit is set"); + "Current number of allocated items if limit is set"); SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, "max_items", CTLFLAG_RD, &zone->uz_max_items, 0, - "Maximum number of cached items"); + "Maximum number of allocated and cached items"); SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, "sleepers", CTLFLAG_RD, &zone->uz_sleepers, 0, "Number of threads sleeping at limit"); @@ -4570,20 +4549,19 @@ zone_free_item(uma_zone_t zone, void *item, void *udat int uma_zone_set_max(uma_zone_t zone, int nitems) { - struct uma_bucket_zone *ubz; - int count; /* + * If the limit is small, we may need to constrain the maximum per-CPU + * cache size, or disable caching entirely. + */ + uma_zone_set_maxcache(zone, nitems); + + /* * XXX This can misbehave if the zone has any allocations with * no limit and a limit is imposed. There is currently no * way to clear a limit. */ ZONE_LOCK(zone); - ubz = bucket_zone_max(zone, nitems); - count = ubz != NULL ? ubz->ubz_entries : 0; - zone->uz_bucket_size_max = zone->uz_bucket_size = count; - if (zone->uz_bucket_size_min > zone->uz_bucket_size_max) - zone->uz_bucket_size_min = zone->uz_bucket_size_max; zone->uz_max_items = nitems; zone->uz_flags |= UMA_ZFLAG_LIMIT; zone_update_caches(zone); @@ -4598,24 +4576,35 @@ uma_zone_set_max(uma_zone_t zone, int nitems) void uma_zone_set_maxcache(uma_zone_t zone, int nitems) { - struct uma_bucket_zone *ubz; - int bpcpu; + int bpcpu, bpdom, bsize, nb; ZONE_LOCK(zone); - ubz = bucket_zone_max(zone, nitems); - if (ubz != NULL) { - bpcpu = 2; - if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) - /* Count the cross-domain bucket. */ - bpcpu++; - nitems -= ubz->ubz_entries * bpcpu * mp_ncpus; - zone->uz_bucket_size_max = ubz->ubz_entries; - } else { - zone->uz_bucket_size_max = zone->uz_bucket_size = 0; + + /* + * Compute a lower bound on the number of items that may be cached in + * the zone. Each CPU gets at least two buckets, and for cross-domain + * frees we use an additional bucket per CPU and per domain. Select the + * largest bucket size that does not exceed half of the requested limit, + * with the left over space given to the full bucket cache. + */ + bpdom = 0; + bpcpu = 2; +#ifdef NUMA + if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && vm_ndomains > 1) { + bpcpu++; + bpdom++; } +#endif + nb = bpcpu * mp_ncpus + bpdom * vm_ndomains; + bsize = nitems / nb / 2; + if (bsize > BUCKET_MAX) + bsize = BUCKET_MAX; + else if (bsize == 0 && nitems / nb > 0) + bsize = 1; + zone->uz_bucket_size_max = zone->uz_bucket_size = bsize; if (zone->uz_bucket_size_min > zone->uz_bucket_size_max) zone->uz_bucket_size_min = zone->uz_bucket_size_max; - zone->uz_bucket_max = nitems / vm_ndomains; + zone->uz_bucket_max = nitems - nb * bsize; ZONE_UNLOCK(zone); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202012062245.0B6MjpvX032617>