From owner-svn-src-all@freebsd.org Fri Oct 30 20:02:33 2020 Return-Path: Delivered-To: svn-src-all@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 9AB6245A740; Fri, 30 Oct 2020 20:02:33 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4CNCs53Snlz4R9L; Fri, 30 Oct 2020 20:02:33 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 5B54E1BBCD; Fri, 30 Oct 2020 20:02:33 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id 09UK2XXq082721; Fri, 30 Oct 2020 20:02:33 GMT (envelope-from mjg@FreeBSD.org) Received: (from mjg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id 09UK2W0j082717; Fri, 30 Oct 2020 20:02:32 GMT (envelope-from mjg@FreeBSD.org) Message-Id: <202010302002.09UK2W0j082717@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mjg set sender to mjg@FreeBSD.org using -f From: Mateusz Guzik Date: Fri, 30 Oct 2020 20:02:32 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r367186 - in head/sys: amd64/amd64 i386/i386 kern sys X-SVN-Group: head X-SVN-Commit-Author: mjg X-SVN-Commit-Paths: in head/sys: amd64/amd64 i386/i386 kern sys X-SVN-Commit-Revision: 367186 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.33 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 30 Oct 2020 20:02:33 -0000 Author: mjg Date: Fri Oct 30 20:02:32 2020 New Revision: 367186 URL: https://svnweb.freebsd.org/changeset/base/367186 Log: malloc: delegate M_EXEC handling to dedicacted routines It is almost never needed and adds an avoidable branch. While here do minior clean ups in preparation for larger changes. Reviewed by: markj Differential Revision: https://reviews.freebsd.org/D27019 Modified: head/sys/amd64/amd64/bpf_jit_machdep.c head/sys/i386/i386/bpf_jit_machdep.c head/sys/kern/kern_malloc.c head/sys/kern/link_elf.c head/sys/sys/malloc.h Modified: head/sys/amd64/amd64/bpf_jit_machdep.c ============================================================================== --- head/sys/amd64/amd64/bpf_jit_machdep.c Fri Oct 30 19:53:16 2020 (r367185) +++ head/sys/amd64/amd64/bpf_jit_machdep.c Fri Oct 30 20:02:32 2020 (r367186) @@ -602,7 +602,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, siz *size = stream.cur_ip; #ifdef _KERNEL - stream.ibuf = malloc(*size, M_BPFJIT, M_EXEC | M_NOWAIT); + stream.ibuf = malloc_exec(*size, M_BPFJIT, M_NOWAIT); if (stream.ibuf == NULL) break; #else Modified: head/sys/i386/i386/bpf_jit_machdep.c ============================================================================== --- head/sys/i386/i386/bpf_jit_machdep.c Fri Oct 30 19:53:16 2020 (r367185) +++ head/sys/i386/i386/bpf_jit_machdep.c Fri Oct 30 20:02:32 2020 (r367186) @@ -632,7 +632,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, siz *size = stream.cur_ip; #ifdef _KERNEL - stream.ibuf = malloc(*size, M_BPFJIT, M_EXEC | M_NOWAIT); + stream.ibuf = malloc_exec(*size, M_BPFJIT, M_NOWAIT); if (stream.ibuf == NULL) break; #else Modified: head/sys/kern/kern_malloc.c ============================================================================== --- head/sys/kern/kern_malloc.c Fri Oct 30 19:53:16 2020 (r367185) +++ head/sys/kern/kern_malloc.c Fri Oct 30 20:02:32 2020 (r367186) @@ -618,13 +618,14 @@ void * unsigned long osize = size; #endif + MPASS((flags & M_EXEC) == 0); #ifdef MALLOC_DEBUG va = NULL; if (malloc_dbg(&va, &size, mtp, flags) != 0) return (va); #endif - if (size <= kmem_zmax && (flags & M_EXEC) == 0) { + if (size <= kmem_zmax) { if (size & KMEM_ZMASK) size = (size & ~KMEM_ZMASK) + KMEM_ZBASE; indx = kmemsize[size >> KMEM_ZSHIFT]; @@ -640,10 +641,11 @@ void * va = malloc_large(&size, DOMAINSET_RR(), flags); malloc_type_allocated(mtp, va == NULL ? 0 : size); } - if (flags & M_WAITOK) - KASSERT(va != NULL, ("malloc(M_WAITOK) returned NULL")); - else if (va == NULL) + if (__predict_false(va == NULL)) { + KASSERT((flags & M_WAITOK) == 0, + ("malloc(M_WAITOK) returned NULL")); t_malloc_fail = time_uptime; + } #ifdef DEBUG_REDZONE if (va != NULL) va = redzone_setup(va, osize); @@ -682,40 +684,102 @@ malloc_domainset(size_t size, struct malloc_type *mtp, int flags) { struct vm_domainset_iter di; - caddr_t ret; + caddr_t va; int domain; int indx; #if defined(DEBUG_REDZONE) unsigned long osize = size; #endif + MPASS((flags & M_EXEC) == 0); #ifdef MALLOC_DEBUG - ret= NULL; - if (malloc_dbg(&ret, &size, mtp, flags) != 0) - return (ret); + va = NULL; + if (malloc_dbg(&va, &size, mtp, flags) != 0) + return (va); #endif - if (size <= kmem_zmax && (flags & M_EXEC) == 0) { + if (size <= kmem_zmax) { vm_domainset_iter_policy_init(&di, ds, &domain, &flags); do { - ret = malloc_domain(&size, &indx, mtp, domain, flags); - } while (ret == NULL && + va = malloc_domain(&size, &indx, mtp, domain, flags); + } while (va == NULL && vm_domainset_iter_policy(&di, &domain) == 0); - malloc_type_zone_allocated(mtp, ret == NULL ? 0 : size, indx); + malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx); } else { /* Policy is handled by kmem. */ - ret = malloc_large(&size, ds, flags); - malloc_type_allocated(mtp, ret == NULL ? 0 : size); + va = malloc_large(&size, ds, flags); + malloc_type_allocated(mtp, va == NULL ? 0 : size); } + if (__predict_false(va == NULL)) { + KASSERT((flags & M_WAITOK) == 0, + ("malloc(M_WAITOK) returned NULL")); + t_malloc_fail = time_uptime; + } +#ifdef DEBUG_REDZONE + if (va != NULL) + va = redzone_setup(va, osize); +#endif + return (va); +} - if (flags & M_WAITOK) - KASSERT(ret != NULL, ("malloc(M_WAITOK) returned NULL")); - else if (ret == NULL) +/* + * Allocate an executable area. + */ +void * +malloc_exec(size_t size, struct malloc_type *mtp, int flags) +{ + caddr_t va; +#if defined(DEBUG_REDZONE) + unsigned long osize = size; +#endif + + flags |= M_EXEC; +#ifdef MALLOC_DEBUG + va = NULL; + if (malloc_dbg(&va, &size, mtp, flags) != 0) + return (va); +#endif + va = malloc_large(&size, DOMAINSET_RR(), flags); + malloc_type_allocated(mtp, va == NULL ? 0 : size); + if (__predict_false(va == NULL)) { + KASSERT((flags & M_WAITOK) == 0, + ("malloc(M_WAITOK) returned NULL")); t_malloc_fail = time_uptime; + } #ifdef DEBUG_REDZONE - if (ret != NULL) - ret = redzone_setup(ret, osize); + if (va != NULL) + va = redzone_setup(va, osize); #endif - return (ret); + return ((void *) va); +} + +void * +malloc_domainset_exec(size_t size, struct malloc_type *mtp, struct domainset *ds, + int flags) +{ + caddr_t va; +#if defined(DEBUG_REDZONE) + unsigned long osize = size; +#endif + + flags |= M_EXEC; +#ifdef MALLOC_DEBUG + va = NULL; + if (malloc_dbg(&va, &size, mtp, flags) != 0) + return (va); +#endif + /* Policy is handled by kmem. */ + va = malloc_large(&size, ds, flags); + malloc_type_allocated(mtp, va == NULL ? 0 : size); + if (__predict_false(va == NULL)) { + KASSERT((flags & M_WAITOK) == 0, + ("malloc(M_WAITOK) returned NULL")); + t_malloc_fail = time_uptime; + } +#ifdef DEBUG_REDZONE + if (va != NULL) + va = redzone_setup(va, osize); +#endif + return (va); } void * Modified: head/sys/kern/link_elf.c ============================================================================== --- head/sys/kern/link_elf.c Fri Oct 30 19:53:16 2020 (r367185) +++ head/sys/kern/link_elf.c Fri Oct 30 20:02:32 2020 (r367186) @@ -1129,7 +1129,7 @@ link_elf_load_file(linker_class_t cls, const char* fil goto out; } #else - mapbase = malloc(mapsize, M_LINKER, M_EXEC | M_WAITOK); + mapbase = malloc_exec(mapsize, M_LINKER, M_WAITOK); #endif ef->address = mapbase; Modified: head/sys/sys/malloc.h ============================================================================== --- head/sys/sys/malloc.h Fri Oct 30 19:53:16 2020 (r367185) +++ head/sys/sys/malloc.h Fri Oct 30 20:02:32 2020 (r367186) @@ -239,6 +239,11 @@ void *malloc_domainset(size_t size, struct malloc_type void *mallocarray(size_t nmemb, size_t size, struct malloc_type *type, int flags) __malloc_like __result_use_check __alloc_size2(1, 2); +void *malloc_exec(size_t size, struct malloc_type *type, int flags) __malloc_like + __result_use_check __alloc_size(1); +void *malloc_domainset_exec(size_t size, struct malloc_type *type, + struct domainset *ds, int flags) __malloc_like __result_use_check + __alloc_size(1); void malloc_init(void *); int malloc_last_fail(void); void malloc_type_allocated(struct malloc_type *type, unsigned long size);