From owner-svn-src-all@freebsd.org Sun Feb 16 01:07:20 2020 Return-Path: Delivered-To: svn-src-all@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 5EE7324EF89; Sun, 16 Feb 2020 01:07:20 +0000 (UTC) (envelope-from jeff@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) server-signature RSA-PSS (4096 bits) client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 48Kpqr11rHz4XpM; Sun, 16 Feb 2020 01:07:19 +0000 (UTC) (envelope-from jeff@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id E375D1DB71; Sun, 16 Feb 2020 01:07:19 +0000 (UTC) (envelope-from jeff@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id 01G17JPL096350; Sun, 16 Feb 2020 01:07:19 GMT (envelope-from jeff@FreeBSD.org) Received: (from jeff@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id 01G17JBm096349; Sun, 16 Feb 2020 01:07:19 GMT (envelope-from jeff@FreeBSD.org) Message-Id: <202002160107.01G17JBm096349@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: jeff set sender to jeff@FreeBSD.org using -f From: Jeff Roberson Date: Sun, 16 Feb 2020 01:07:19 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r357988 - head/sys/vm X-SVN-Group: head X-SVN-Commit-Author: jeff X-SVN-Commit-Paths: head/sys/vm X-SVN-Commit-Revision: 357988 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 16 Feb 2020 01:07:20 -0000 Author: jeff Date: Sun Feb 16 01:07:19 2020 New Revision: 357988 URL: https://svnweb.freebsd.org/changeset/base/357988 Log: Slightly restructure uma_zalloc* to generate better code from clang and reduce duplication among zalloc functions. Reviewed by: markj Discussed with: mjg Differential Revision: https://reviews.freebsd.org/D23672 Modified: head/sys/vm/uma_core.c Modified: head/sys/vm/uma_core.c ============================================================================== --- head/sys/vm/uma_core.c Sun Feb 16 00:12:53 2020 (r357987) +++ head/sys/vm/uma_core.c Sun Feb 16 01:07:19 2020 (r357988) @@ -3013,8 +3013,8 @@ item_ctor(uma_zone_t zone, int uz_flags, int size, voi if (!skipdbg) uma_dbg_alloc(zone, NULL, item); #endif - if (flags & M_ZERO) - bzero(item, size); + if (__predict_false(flags & M_ZERO)) + return (memset(item, 0, size)); return (item); } @@ -3117,11 +3117,35 @@ uma_zfree_debug(uma_zone_t zone, void *item, void *uda } #endif +static inline void * +cache_alloc_item(uma_zone_t zone, uma_cache_t cache, uma_cache_bucket_t bucket, + void *udata, int flags) +{ + void *item; + int size, uz_flags; + + item = cache_bucket_pop(cache, bucket); + size = cache_uz_size(cache); + uz_flags = cache_uz_flags(cache); + critical_exit(); + return (item_ctor(zone, uz_flags, size, udata, flags, item)); +} + static __noinline void * -uma_zalloc_single(uma_zone_t zone, void *udata, int flags) +cache_alloc_retry(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) { + uma_cache_bucket_t bucket; int domain; + while (cache_alloc(zone, cache, udata, flags)) { + cache = &zone->uz_cpu[curcpu]; + bucket = &cache->uc_allocbucket; + if (__predict_false(bucket->ucb_cnt == 0)) + continue; + return (cache_alloc_item(zone, cache, bucket, udata, flags)); + } + critical_exit(); + /* * We can not get a bucket so try to return a single item. */ @@ -3138,10 +3162,10 @@ uma_zalloc_smr(uma_zone_t zone, int flags) { uma_cache_bucket_t bucket; uma_cache_t cache; - void *item; - int size, uz_flags; #ifdef UMA_ZALLOC_DEBUG + void *item; + KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, ("uma_zalloc_arg: called with non-SMR zone.\n")); if (uma_zalloc_debug(zone, &item, NULL, flags) == EJUSTRETURN) @@ -3149,21 +3173,11 @@ uma_zalloc_smr(uma_zone_t zone, int flags) #endif critical_enter(); - do { - cache = &zone->uz_cpu[curcpu]; - bucket = &cache->uc_allocbucket; - size = cache_uz_size(cache); - uz_flags = cache_uz_flags(cache); - if (__predict_true(bucket->ucb_cnt != 0)) { - item = cache_bucket_pop(cache, bucket); - critical_exit(); - return (item_ctor(zone, uz_flags, size, NULL, flags, - item)); - } - } while (cache_alloc(zone, cache, NULL, flags)); - critical_exit(); - - return (uma_zalloc_single(zone, NULL, flags)); + cache = &zone->uz_cpu[curcpu]; + bucket = &cache->uc_allocbucket; + if (__predict_false(bucket->ucb_cnt == 0)) + return (cache_alloc_retry(zone, cache, NULL, flags)); + return (cache_alloc_item(zone, cache, bucket, NULL, flags)); } /* See uma.h */ @@ -3172,8 +3186,6 @@ uma_zalloc_arg(uma_zone_t zone, void *udata, int flags { uma_cache_bucket_t bucket; uma_cache_t cache; - void *item; - int size, uz_flags; /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); @@ -3183,6 +3195,8 @@ uma_zalloc_arg(uma_zone_t zone, void *udata, int flags zone, flags); #ifdef UMA_ZALLOC_DEBUG + void *item; + KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, ("uma_zalloc_arg: called with SMR zone.\n")); if (uma_zalloc_debug(zone, &item, udata, flags) == EJUSTRETURN) @@ -3201,21 +3215,11 @@ uma_zalloc_arg(uma_zone_t zone, void *udata, int flags * must detect and handle migration if it has occurred. */ critical_enter(); - do { - cache = &zone->uz_cpu[curcpu]; - bucket = &cache->uc_allocbucket; - size = cache_uz_size(cache); - uz_flags = cache_uz_flags(cache); - if (__predict_true(bucket->ucb_cnt != 0)) { - item = cache_bucket_pop(cache, bucket); - critical_exit(); - return (item_ctor(zone, uz_flags, size, udata, flags, - item)); - } - } while (cache_alloc(zone, cache, udata, flags)); - critical_exit(); - - return (uma_zalloc_single(zone, udata, flags)); + cache = &zone->uz_cpu[curcpu]; + bucket = &cache->uc_allocbucket; + if (__predict_false(bucket->ucb_cnt == 0)) + return (cache_alloc_retry(zone, cache, udata, flags)); + return (cache_alloc_item(zone, cache, bucket, udata, flags)); } /* @@ -4326,7 +4330,7 @@ zone_release(void *arg, void **bucket, int cnt) * udata User supplied data for the dtor * skip Skip dtors and finis */ -static void +static __noinline void zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) {