Date: Thu, 6 Jun 2019 23:57:28 +0000 (UTC) From: Alexander Motin <mav@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r348764 - head/sys/vm Message-ID: <201906062357.x56NvSp7013739@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: mav Date: Thu Jun 6 23:57:28 2019 New Revision: 348764 URL: https://svnweb.freebsd.org/changeset/base/348764 Log: Allow UMA hash tables to expand faster then 2x in 20 seconds. ZFS ABD allocates tons of 4KB chunks via UMA, requiring huge hash tables. With initial hash table size of only 32 elements it takes ~20 expansions or ~400 seconds to adapt to handling 220GB ZFS ARC. During that time not only the hash table is highly inefficient, but also each of those expan- sions takes significant time with the lock held, blocking operation. On my test system with 256GB of RAM and ZFS pool of 28 HDDs this change reduces time needed to first time read 240GB from ~300-400s, during which system is quite busy and unresponsive, to only ~150s with light CPU load and just 5 sub-second CPU spikes to expand the hash table. MFC after: 2 weeks Sponsored by: iXsystems, Inc. Modified: head/sys/vm/uma_core.c Modified: head/sys/vm/uma_core.c ============================================================================== --- head/sys/vm/uma_core.c Thu Jun 6 22:56:55 2019 (r348763) +++ head/sys/vm/uma_core.c Thu Jun 6 23:57:28 2019 (r348764) @@ -261,7 +261,7 @@ static void keg_small_init(uma_keg_t keg); static void keg_large_init(uma_keg_t keg); static void zone_foreach(void (*zfunc)(uma_zone_t)); static void zone_timeout(uma_zone_t zone); -static int hash_alloc(struct uma_hash *); +static int hash_alloc(struct uma_hash *, u_int); static int hash_expand(struct uma_hash *, struct uma_hash *); static void hash_free(struct uma_hash *hash); static void uma_timeout(void *); @@ -572,6 +572,7 @@ static void zone_timeout(uma_zone_t zone) { uma_keg_t keg = zone->uz_keg; + u_int slabs; KEG_LOCK(keg); /* @@ -582,7 +583,8 @@ zone_timeout(uma_zone_t zone) * may be a little aggressive. Should I allow for two collisions max? */ if (keg->uk_flags & UMA_ZONE_HASH && - keg->uk_pages / keg->uk_ppera >= keg->uk_hash.uh_hashsize) { + (slabs = keg->uk_pages / keg->uk_ppera) > + keg->uk_hash.uh_hashsize) { struct uma_hash newhash; struct uma_hash oldhash; int ret; @@ -593,9 +595,8 @@ zone_timeout(uma_zone_t zone) * I have to do everything in stages and check for * races. */ - newhash = keg->uk_hash; KEG_UNLOCK(keg); - ret = hash_alloc(&newhash); + ret = hash_alloc(&newhash, 1 << fls(slabs)); KEG_LOCK(keg); if (ret) { if (hash_expand(&keg->uk_hash, &newhash)) { @@ -627,16 +628,13 @@ zone_timeout(uma_zone_t zone) * 1 on success and 0 on failure. */ static int -hash_alloc(struct uma_hash *hash) +hash_alloc(struct uma_hash *hash, u_int size) { - u_int oldsize; size_t alloc; - oldsize = hash->uh_hashsize; - - /* We're just going to go to a power of two greater */ - if (oldsize) { - hash->uh_hashsize = oldsize * 2; + KASSERT(powerof2(size), ("hash size must be power of 2")); + if (size > UMA_HASH_SIZE_INIT) { + hash->uh_hashsize = size; alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize; hash->uh_slab_hash = (struct slabhead *)malloc(alloc, M_UMAHASH, M_NOWAIT); @@ -1711,7 +1709,7 @@ keg_ctor(void *mem, int size, void *udata, int flags) } if (keg->uk_flags & UMA_ZONE_HASH) - hash_alloc(&keg->uk_hash); + hash_alloc(&keg->uk_hash, 0); CTR5(KTR_UMA, "keg_ctor %p zone %s(%p) out %d free %d\n", keg, zone->uz_name, zone,
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201906062357.x56NvSp7013739>