Date: Fri, 22 Jun 2018 23:08:28 -1000 (HST) From: Jeff Roberson <jroberson@jroberson.net> To: Jeff Roberson <jeff@FreeBSD.org> Cc: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: Re: svn commit: r335579 - head/sys/vm Message-ID: <alpine.BSF.2.21.999.1806222308040.2934@desktop> In-Reply-To: <201806230810.w5N8AAr2037440@repo.freebsd.org> References: <201806230810.w5N8AAr2037440@repo.freebsd.org>
next in thread | previous in thread | raw e-mail | index | archive | help
On Sat, 23 Jun 2018, Jeff Roberson wrote: > Author: jeff > Date: Sat Jun 23 08:10:09 2018 > New Revision: 335579 > URL: https://svnweb.freebsd.org/changeset/base/335579 > > Log: > Sort uma_zone fields according to 64 byte cache line with adjacent line > prefetch on 64bit architectures. Prior to this, two lines were needed > for the fast path and each line may fetch an unused adjacent neighbor. > - Move fields used by the fast path into a single line. > - Move constants into the adjacent line which is mostly used for > the spare bucket alloc 'medium path'. > - Unpad the mtx which is only used by the fast path and place it in I got a little overexcited about fast. The mtx is only used by the slow path. Jeff > a line with rarely used data. This aligns the cachelines better and > eliminates 128 bytes of wasted space. > > This gives a 45% improvement on a will-it-scale test on a 24 core machine. > > Reviewed by: mmacy > > Modified: > head/sys/vm/uma_int.h > > Modified: head/sys/vm/uma_int.h > ============================================================================== > --- head/sys/vm/uma_int.h Sat Jun 23 07:14:08 2018 (r335578) > +++ head/sys/vm/uma_int.h Sat Jun 23 08:10:09 2018 (r335579) > @@ -222,9 +222,8 @@ typedef struct uma_domain * uma_domain_t; > * > */ > struct uma_keg { > - struct mtx_padalign uk_lock; /* Lock for the keg */ > + struct mtx uk_lock; /* Lock for the keg */ > struct uma_hash uk_hash; > - > LIST_HEAD(,uma_zone) uk_zones; /* Keg's zones */ > > uint32_t uk_cursor; /* Domain alloc cursor. */ > @@ -315,40 +314,48 @@ typedef struct uma_zone_domain * uma_zone_domain_t; > * > */ > struct uma_zone { > - struct mtx_padalign uz_lock; /* Lock for the zone */ > - struct mtx_padalign *uz_lockptr; > - const char *uz_name; /* Text name of the zone */ > - > - LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */ > + /* Offset 0, used in alloc/free fast/medium fast path and const. */ > + struct mtx *uz_lockptr; > + const char *uz_name; /* Text name of the zone */ > struct uma_zone_domain *uz_domain; /* per-domain buckets */ > - > - LIST_HEAD(,uma_klink) uz_kegs; /* List of kegs. */ > - struct uma_klink uz_klink; /* klink for first keg. */ > - > - uma_slaballoc uz_slab; /* Allocate a slab from the backend. */ > + uint32_t uz_flags; /* Flags inherited from kegs */ > + uint32_t uz_size; /* Size inherited from kegs */ > uma_ctor uz_ctor; /* Constructor for each allocation */ > uma_dtor uz_dtor; /* Destructor */ > uma_init uz_init; /* Initializer for each item */ > uma_fini uz_fini; /* Finalizer for each item. */ > + > + /* Offset 64, used in bucket replenish. */ > uma_import uz_import; /* Import new memory to cache. */ > uma_release uz_release; /* Release memory from cache. */ > void *uz_arg; /* Import/release argument. */ > - > - uint32_t uz_flags; /* Flags inherited from kegs */ > - uint32_t uz_size; /* Size inherited from kegs */ > - > - volatile u_long uz_allocs UMA_ALIGN; /* Total number of allocations */ > - volatile u_long uz_fails; /* Total number of alloc failures */ > - volatile u_long uz_frees; /* Total number of frees */ > - uint64_t uz_sleeps; /* Total number of alloc sleeps */ > + uma_slaballoc uz_slab; /* Allocate a slab from the backend. */ > uint16_t uz_count; /* Amount of items in full bucket */ > uint16_t uz_count_min; /* Minimal amount of items there */ > + /* 32bit pad on 64bit. */ > + LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */ > + LIST_HEAD(,uma_klink) uz_kegs; /* List of kegs. */ > > + /* Offset 128 Rare. */ > + /* > + * The lock is placed here to avoid adjacent line prefetcher > + * in fast paths and to take up space near infrequently accessed > + * members to reduce alignment overhead. > + */ > + struct mtx uz_lock; /* Lock for the zone */ > + struct uma_klink uz_klink; /* klink for first keg. */ > /* The next two fields are used to print a rate-limited warnings. */ > const char *uz_warning; /* Warning to print on failure */ > struct timeval uz_ratecheck; /* Warnings rate-limiting */ > - > struct task uz_maxaction; /* Task to run when at limit */ > + > + /* 16 bytes of pad. */ > + > + /* Offset 256, atomic stats. */ > + volatile u_long uz_allocs UMA_ALIGN; /* Total number of allocations */ > + volatile u_long uz_fails; /* Total number of alloc failures */ > + volatile u_long uz_frees; /* Total number of frees */ > + uint64_t uz_sleeps; /* Total number of alloc sleeps */ > > /* > * This HAS to be the last item because we adjust the zone size >
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?alpine.BSF.2.21.999.1806222308040.2934>