From owner-svn-src-head@freebsd.org Mon Nov 16 17:56:59 2020 Return-Path: Delivered-To: svn-src-head@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 1EDE146D191; Mon, 16 Nov 2020 17:56:59 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4CZcGM0NVxz3vl5; Mon, 16 Nov 2020 17:56:59 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id F3D2E11C7F; Mon, 16 Nov 2020 17:56:58 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id 0AGHuwlE043447; Mon, 16 Nov 2020 17:56:58 GMT (envelope-from mjg@FreeBSD.org) Received: (from mjg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id 0AGHuwwH043446; Mon, 16 Nov 2020 17:56:58 GMT (envelope-from mjg@FreeBSD.org) Message-Id: <202011161756.0AGHuwwH043446@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mjg set sender to mjg@FreeBSD.org using -f From: Mateusz Guzik Date: Mon, 16 Nov 2020 17:56:58 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r367733 - head/sys/kern X-SVN-Group: head X-SVN-Commit-Author: mjg X-SVN-Commit-Paths: head/sys/kern X-SVN-Commit-Revision: 367733 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 16 Nov 2020 17:56:59 -0000 Author: mjg Date: Mon Nov 16 17:56:58 2020 New Revision: 367733 URL: https://svnweb.freebsd.org/changeset/base/367733 Log: malloc: make malloc_large closer to standalone This moves entire large alloc handling out of all consumers, apart from deciding to go there. This is a step towards creating a fast path. Reviewed by: markj Differential Revision: https://reviews.freebsd.org/D27198 Modified: head/sys/kern/kern_malloc.c Modified: head/sys/kern/kern_malloc.c ============================================================================== --- head/sys/kern/kern_malloc.c Mon Nov 16 17:45:42 2020 (r367732) +++ head/sys/kern/kern_malloc.c Mon Nov 16 17:56:58 2020 (r367733) @@ -110,6 +110,14 @@ dtrace_malloc_probe_func_t __read_mostly dtrace_malloc #define MALLOC_DEBUG 1 #endif +#ifdef DEBUG_REDZONE +#define DEBUG_REDZONE_ARG_DEF , unsigned long osize +#define DEBUG_REDZONE_ARG , osize +#else +#define DEBUG_REDZONE_ARG_DEF +#define DEBUG_REDZONE_ARG +#endif + /* * When realloc() is called, if the new size is sufficiently smaller than * the old size, realloc() will allocate a new, smaller block to avoid @@ -574,21 +582,33 @@ malloc_large_size(uma_slab_t slab) return (va >> 1); } -static caddr_t -malloc_large(size_t *size, struct domainset *policy, int flags) +static caddr_t __noinline +malloc_large(size_t *size, struct malloc_type *mtp, struct domainset *policy, + int flags DEBUG_REDZONE_ARG_DEF) { - vm_offset_t va; + vm_offset_t kva; + caddr_t va; size_t sz; sz = roundup(*size, PAGE_SIZE); - va = kmem_malloc_domainset(policy, sz, flags); - if (va != 0) { + kva = kmem_malloc_domainset(policy, sz, flags); + if (kva != 0) { /* The low bit is unused for slab pointers. */ - vsetzoneslab(va, NULL, (void *)((sz << 1) | 1)); + vsetzoneslab(kva, NULL, (void *)((sz << 1) | 1)); uma_total_inc(sz); *size = sz; } - return ((caddr_t)va); + va = (caddr_t)kva; + malloc_type_allocated(mtp, va == NULL ? 0 : sz); + if (__predict_false(va == NULL)) { + KASSERT((flags & M_WAITOK) == 0, + ("malloc(M_WAITOK) returned NULL")); + } +#ifdef DEBUG_REDZONE + if (va != NULL) + va = redzone_setup(va, osize); +#endif + return (va); } static void @@ -613,30 +633,30 @@ void * int indx; caddr_t va; uma_zone_t zone; -#if defined(DEBUG_REDZONE) +#ifdef DEBUG_REDZONE unsigned long osize = size; #endif MPASS((flags & M_EXEC) == 0); + #ifdef MALLOC_DEBUG va = NULL; if (malloc_dbg(&va, &size, mtp, flags) != 0) return (va); #endif - if (size <= kmem_zmax) { - if (size & KMEM_ZMASK) - size = (size & ~KMEM_ZMASK) + KMEM_ZBASE; - indx = kmemsize[size >> KMEM_ZSHIFT]; - zone = kmemzones[indx].kz_zone[mtp_get_subzone(mtp)]; - va = uma_zalloc(zone, flags); - if (va != NULL) - size = zone->uz_size; - malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx); - } else { - va = malloc_large(&size, DOMAINSET_RR(), flags); - malloc_type_allocated(mtp, va == NULL ? 0 : size); - } + if (__predict_false(size > kmem_zmax)) + return (malloc_large(&size, mtp, DOMAINSET_RR(), flags + DEBUG_REDZONE_ARG)); + + if (size & KMEM_ZMASK) + size = (size & ~KMEM_ZMASK) + KMEM_ZBASE; + indx = kmemsize[size >> KMEM_ZSHIFT]; + zone = kmemzones[indx].kz_zone[mtp_get_subzone(mtp)]; + va = uma_zalloc(zone, flags); + if (va != NULL) + size = zone->uz_size; + malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx); if (__predict_false(va == NULL)) { KASSERT((flags & M_WAITOK) == 0, ("malloc(M_WAITOK) returned NULL")); @@ -679,28 +699,27 @@ malloc_domainset(size_t size, struct malloc_type *mtp, caddr_t va; int domain; int indx; - -#if defined(DEBUG_REDZONE) +#ifdef DEBUG_REDZONE unsigned long osize = size; #endif + MPASS((flags & M_EXEC) == 0); + #ifdef MALLOC_DEBUG va = NULL; if (malloc_dbg(&va, &size, mtp, flags) != 0) return (va); #endif - if (size <= kmem_zmax) { - vm_domainset_iter_policy_init(&di, ds, &domain, &flags); - do { - va = malloc_domain(&size, &indx, mtp, domain, flags); - } while (va == NULL && - vm_domainset_iter_policy(&di, &domain) == 0); - malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx); - } else { - /* Policy is handled by kmem. */ - va = malloc_large(&size, ds, flags); - malloc_type_allocated(mtp, va == NULL ? 0 : size); - } + + if (__predict_false(size > kmem_zmax)) + return (malloc_large(&size, mtp, DOMAINSET_RR(), flags + DEBUG_REDZONE_ARG)); + + vm_domainset_iter_policy_init(&di, ds, &domain, &flags); + do { + va = malloc_domain(&size, &indx, mtp, domain, flags); + } while (va == NULL && vm_domainset_iter_policy(&di, &domain) == 0); + malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx); if (__predict_false(va == NULL)) { KASSERT((flags & M_WAITOK) == 0, ("malloc(M_WAITOK) returned NULL")); @@ -718,57 +737,30 @@ malloc_domainset(size_t size, struct malloc_type *mtp, void * malloc_exec(size_t size, struct malloc_type *mtp, int flags) { - caddr_t va; -#if defined(DEBUG_REDZONE) - unsigned long osize = size; -#endif - flags |= M_EXEC; -#ifdef MALLOC_DEBUG - va = NULL; - if (malloc_dbg(&va, &size, mtp, flags) != 0) - return (va); -#endif - va = malloc_large(&size, DOMAINSET_RR(), flags); - malloc_type_allocated(mtp, va == NULL ? 0 : size); - if (__predict_false(va == NULL)) { - KASSERT((flags & M_WAITOK) == 0, - ("malloc(M_WAITOK) returned NULL")); - } -#ifdef DEBUG_REDZONE - if (va != NULL) - va = redzone_setup(va, osize); -#endif - return ((void *) va); + return (malloc_domainset_exec(size, mtp, DOMAINSET_RR(), flags)); } void * malloc_domainset_exec(size_t size, struct malloc_type *mtp, struct domainset *ds, int flags) { - caddr_t va; -#if defined(DEBUG_REDZONE) +#ifdef DEBUG_REDZONE unsigned long osize = size; #endif +#ifdef MALLOC_DEBUG + caddr_t va; +#endif flags |= M_EXEC; + #ifdef MALLOC_DEBUG va = NULL; if (malloc_dbg(&va, &size, mtp, flags) != 0) return (va); #endif - /* Policy is handled by kmem. */ - va = malloc_large(&size, ds, flags); - malloc_type_allocated(mtp, va == NULL ? 0 : size); - if (__predict_false(va == NULL)) { - KASSERT((flags & M_WAITOK) == 0, - ("malloc(M_WAITOK) returned NULL")); - } -#ifdef DEBUG_REDZONE - if (va != NULL) - va = redzone_setup(va, osize); -#endif - return (va); + + return (malloc_large(&size, mtp, ds, flags DEBUG_REDZONE_ARG)); } void *