Date: Tue, 3 Jul 2018 01:55:10 +0000 (UTC) From: Matt Macy <mmacy@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r335879 - in head/sys: conf kern sys Message-ID: <201807030155.w631tARo009470@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: mmacy Date: Tue Jul 3 01:55:09 2018 New Revision: 335879 URL: https://svnweb.freebsd.org/changeset/base/335879 Log: make critical_{enter, exit} inline Avoid pulling in all of the <sys/proc.h> dependencies by automatically generating a stripped down thread_lite exporting only the fields of interest. The field declarations are type checked against the original and the offsets of the generated result is automatically checked. kib has expressed disagreement and would have preferred to simply use genassym style offsets (which loses type check enforcement). jhb has expressed dislike of it due to header pollution and a duplicate structure. He would have preferred to just have defined thread in _thread.h. Nonetheless, he admits that this is the only viable solution at the moment. The impetus for this came from mjg's D15331: "Inline critical_enter/exit for amd64" Reviewed by: jeff Differential Revision: https://reviews.freebsd.org/D16078 Added: head/sys/kern/genoffset.c (contents, props changed) head/sys/kern/genoffset.sh (contents, props changed) head/sys/sys/kpilite.h (contents, props changed) Modified: head/sys/conf/kern.post.mk head/sys/conf/kern.pre.mk head/sys/kern/kern_switch.c head/sys/sys/assym.h head/sys/sys/systm.h Modified: head/sys/conf/kern.post.mk ============================================================================== --- head/sys/conf/kern.post.mk Mon Jul 2 22:59:29 2018 (r335878) +++ head/sys/conf/kern.post.mk Tue Jul 3 01:55:09 2018 (r335879) @@ -185,13 +185,25 @@ hack.pico: Makefile ${CC} ${HACK_EXTRA_FLAGS} -nostdlib hack.c -o hack.pico rm -f hack.c -assym.inc: $S/kern/genassym.sh genassym.o +offset.inc: $S/kern/genoffset.sh genoffset.o + NM='${NM}' NMFLAGS='${NMFLAGS}' sh $S/kern/genoffset.sh genoffset.o > ${.TARGET} + +genoffset.o: $S/kern/genoffset.c + ${CC} -c ${CFLAGS:N-flto:N-fno-common} $S/kern/genoffset.c + +genoffset_test.c: $S/kern/genoffset.c + cp $S/kern/genoffset.c genoffset_test.c + +genoffset_test.o: genoffset_test.c offset.inc + ${CC} -c ${CFLAGS:N-flto:N-fno-common} -DOFFSET_TEST genoffset_test.c + +assym.inc: $S/kern/genassym.sh genassym.o genoffset_test.o NM='${NM}' NMFLAGS='${NMFLAGS}' sh $S/kern/genassym.sh genassym.o > ${.TARGET} -genassym.o: $S/$M/$M/genassym.c +genassym.o: $S/$M/$M/genassym.c offset.inc ${CC} -c ${CFLAGS:N-flto:N-fno-common} $S/$M/$M/genassym.c -${SYSTEM_OBJS} genassym.o vers.o: opt_global.h +${SYSTEM_OBJS} genoffset.o genassym.o vers.o: opt_global.h .if !empty(.MAKE.MODE:Unormal:Mmeta) && empty(.MAKE.MODE:Unormal:Mnofilemon) _meta_filemon= 1 @@ -213,10 +225,10 @@ _SKIP_DEPEND= 1 .endif kernel-depend: .depend -SRCS= assym.inc vnode_if.h ${BEFORE_DEPEND} ${CFILES} \ +SRCS= assym.inc offset.inc vnode_if.h ${BEFORE_DEPEND} ${CFILES} \ ${SYSTEM_CFILES} ${GEN_CFILES} ${SFILES} \ ${MFILES:T:S/.m$/.h/} -DEPENDOBJS+= ${SYSTEM_OBJS} genassym.o +DEPENDOBJS+= ${SYSTEM_OBJS} genassym.o genoffset.o DEPENDFILES= ${DEPENDOBJS:O:u:C/^/.depend./} .if ${MAKE_VERSION} < 20160220 DEPEND_MP?= -MP Modified: head/sys/conf/kern.pre.mk ============================================================================== --- head/sys/conf/kern.pre.mk Mon Jul 2 22:59:29 2018 (r335878) +++ head/sys/conf/kern.pre.mk Tue Jul 3 01:55:09 2018 (r335879) @@ -195,7 +195,7 @@ OFEDCFLAGS= ${CFLAGS:N-I*} -DCONFIG_INFINIBAND_USER_ME OFED_C_NOIMP= ${CC} -c -o ${.TARGET} ${OFEDCFLAGS} ${WERROR} ${PROF} OFED_C= ${OFED_C_NOIMP} ${.IMPSRC} -GEN_CFILES= $S/$M/$M/genassym.c ${MFILES:T:S/.m$/.c/} +GEN_CFILES= $S/$M/$M/genassym.c $S/kern/genoffset.c ${MFILES:T:S/.m$/.c/} SYSTEM_CFILES= config.c env.c hints.c vnode_if.c SYSTEM_DEP= Makefile ${SYSTEM_OBJS} SYSTEM_OBJS= locore.o ${MDOBJS} ${OBJS} Added: head/sys/kern/genoffset.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/kern/genoffset.c Tue Jul 3 01:55:09 2018 (r335879) @@ -0,0 +1,44 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef OFFSET_TEST +#define GENOFFSET +#endif +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); +#include <sys/param.h> +#include <sys/assym.h> +#include <sys/proc.h> + +OFFSYM(td_pre_epoch_prio, thread, u_char); +OFFSYM(td_priority, thread, u_char); +OFFSYM(td_epochnest, thread, u_char); +OFFSYM(td_critnest, thread, u_int); +OFFSYM(td_pinned, thread, int); +OFFSYM(td_owepreempt, thread, u_char); Added: head/sys/kern/genoffset.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/kern/genoffset.sh Tue Jul 3 01:55:09 2018 (r335879) @@ -0,0 +1,141 @@ +#!/bin/sh + +# SPDX-License-Identifier: BSD-2-Clause-FreeBSD +# +# Copyright (c) 2000, Bruce Evans <bde@freebsd.org> +# Copyright (c) 2018, Jeff Roberson <jeff@freebsd.org> +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ + +usage() +{ + echo "usage: genoffset [-o outfile] objfile" + exit 1 +} + + +work() +{ + echo "#ifndef _OFFSET_INC_" + echo "#define _OFFSET_INC_" + echo "#if !defined(GENOFFSET) && !defined(KLD_MODULE)" + ${NM:='nm'} ${NMFLAGS} "$1" | ${AWK:='awk'} ' + / C .*_datatype_*/ { + type = substr($3, match($3, "_datatype_") + length("_datatype_")) + } + / C .*_parenttype_*/ { + parent = substr($3, match($3, "_parenttype_") + length("_parenttype_")) + } + / C .*sign$/ { + sign = substr($1, length($1) - 3, 4) + sub("^0*", "", sign) + if (sign != "") + sign = "-" + } + / C .*w0$/ { + w0 = substr($1, length($1) - 3, 4) + } + / C .*w1$/ { + w1 = substr($1, length($1) - 3, 4) + } + / C .*w2$/ { + w2 = substr($1, length($1) - 3, 4) + } + / C .*w3$/ { + w3 = substr($1, length($1) - 3, 4) + w = w3 w2 w1 w0 + sub("^0*", "", w) + if (w == "") + w = "0" + hex = "" + if (w != "0") + hex = "0x" + sub("w3$", "", $3) + member = tolower($3) + # This still has minor problems representing INT_MIN, etc. + # E.g., + # with 32-bit 2''s complement ints, this prints -0x80000000, + # which has the wrong type (unsigned int). + offset = sprintf("%s%s%s", sign, hex, w) + + structures[parent] = sprintf("%s%s %s %s\n", + structures[parent], offset, type, member) + } + END { + for (struct in structures) { + printf("struct %s_lite {\n", struct); + n = split(structures[struct], members, "\n") + for (i = 1; i < n; i++) { + for (j = i + 1; j < n; j++) { + split(members[i], ivar, " ") + split(members[j], jvar, " ") + if (jvar[1] < ivar[1]) { + tmp = members[i] + members[i] = members[j] + members[j] = tmp + } + } + } + off = "0" + for (i = 1; i < n; i++) { + split(members[i], m, " ") + printf "\tu_char\tpad_%s[%s - %s];\n", m[3], m[1], off + printf "\t%s\t%s;\n", m[2], m[3] + off = sprintf("(%s + sizeof(%s))", m[1], m[2]) + } + printf("};\n"); + } + } + ' + + echo "#endif" + echo "#endif" +} + + +# +#MAIN PROGGRAM +# +use_outfile="no" +while getopts "o:" option +do + case "$option" in + o) outfile="$OPTARG" + use_outfile="yes";; + *) usage;; + esac +done +shift $(($OPTIND - 1)) +case $# in +1) ;; +*) usage;; +esac + +if [ "$use_outfile" = "yes" ] +then + work $1 3>"$outfile" >&3 3>&- +else + work $1 +fi + Modified: head/sys/kern/kern_switch.c ============================================================================== --- head/sys/kern/kern_switch.c Mon Jul 2 22:59:29 2018 (r335878) +++ head/sys/kern/kern_switch.c Tue Jul 3 01:55:09 2018 (r335879) @@ -199,17 +199,17 @@ choosethread(void) * the function call itself, for most cases. */ void -critical_enter(void) +critical_enter_KBI(void) { - struct thread *td; - - td = curthread; - td->td_critnest++; +#ifdef KTR + struct thread *td = curthread; +#endif + critical_enter(); CTR4(KTR_CRITICAL, "critical_enter by thread %p (%ld, %s) to %d", td, (long)td->td_proc->p_pid, td->td_name, td->td_critnest); } -static void __noinline +void __noinline critical_exit_preempt(void) { struct thread *td; @@ -245,17 +245,12 @@ critical_exit_preempt(void) } void -critical_exit(void) +critical_exit_KBI(void) { - struct thread *td; - - td = curthread; - KASSERT(td->td_critnest != 0, - ("critical_exit: td_critnest == 0")); - td->td_critnest--; - __compiler_membar(); - if (__predict_false(td->td_owepreempt)) - critical_exit_preempt(); +#ifdef KTR + struct thread *td = curthread; +#endif + critical_exit(); CTR4(KTR_CRITICAL, "critical_exit by thread %p (%ld, %s) to %d", td, (long)td->td_proc->p_pid, td->td_name, td->td_critnest); } Modified: head/sys/sys/assym.h ============================================================================== --- head/sys/sys/assym.h Mon Jul 2 22:59:29 2018 (r335878) +++ head/sys/sys/assym.h Tue Jul 3 01:55:09 2018 (r335879) @@ -43,4 +43,19 @@ char name ## w1[((ASSYM_ABS(value) & 0xFFFF0000UL) >> char name ## w2[((ASSYM_ABS(value) & 0xFFFF00000000ULL) >> 32) + ASSYM_BIAS]; \ char name ## w3[((ASSYM_ABS(value) & 0xFFFF000000000000ULL) >> 48) + ASSYM_BIAS] + +/* char name ## _datatype_ ## STRINGIFY(typeof(((struct parenttype *)(0x0))-> name)) [1]; */ +#ifdef OFFSET_TEST +#define OFFSET_CTASSERT CTASSERT +#else +#define OFFSET_CTASSERT(...) +#endif + +#define OFFSYM(name, parenttype, datatype) \ +ASSYM(name, offsetof(struct parenttype, name)); \ +char name ## _datatype_ ## datatype [1]; \ +char name ## _parenttype_ ## parenttype [1]; \ +CTASSERT(__builtin_types_compatible_p(__typeof(((struct parenttype *)(0x0))-> name), datatype)); \ +OFFSET_CTASSERT(offsetof(struct parenttype, name) == offsetof(struct parenttype ## _lite, name)) + #endif /* !_SYS_ASSYM_H_ */ Added: head/sys/sys/kpilite.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/sys/kpilite.h Tue Jul 3 01:55:09 2018 (r335879) @@ -0,0 +1,55 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS_KPILITE_H_ +#define _SYS_KPILITE_H_ +#if !defined(GENOFFSET) && !defined(KLD_MODULE) && defined(_KERNEL) +#include "offset.inc" + +static __inline void +sched_pin_lite(struct thread_lite *td) +{ + + KASSERT((struct thread *)td == curthread, ("sched_pin called on non curthread")); + td->td_pinned++; + __compiler_membar(); +} + +static __inline void +sched_unpin_lite(struct thread_lite *td) +{ + + KASSERT((struct thread *)td == curthread, ("sched_unpin called on non curthread")); + KASSERT(td->td_pinned > 0, ("sched_unpin called on non pinned thread")); + __compiler_membar(); + td->td_pinned--; + __compiler_membar(); +} +#endif +#endif Modified: head/sys/sys/systm.h ============================================================================== --- head/sys/sys/systm.h Mon Jul 2 22:59:29 2018 (r335878) +++ head/sys/sys/systm.h Tue Jul 3 01:55:09 2018 (r335879) @@ -107,6 +107,12 @@ void kassert_panic(const char *fmt, ...) __printflike #define CTASSERT(x) _Static_assert(x, "compile-time assertion failed") #endif +#if defined(_KERNEL) +#include <sys/param.h> /* MAXCPU */ +#include <sys/pcpu.h> /* curthread */ +#include <sys/kpilite.h> +#endif + /* * Assert that a pointer can be loaded from memory atomically. * @@ -214,12 +220,44 @@ void vpanic(const char *, __va_list) __dead2 __printfl void cpu_boot(int); void cpu_flush_dcache(void *, size_t); void cpu_rootconf(void); -void critical_enter(void); -void critical_exit(void); +void critical_enter_KBI(void); +void critical_exit_KBI(void); +void critical_exit_preempt(void); void init_param1(void); void init_param2(long physpages); void init_static_kenv(char *, size_t); void tablefull(const char *); + +#if defined(KLD_MODULE) || defined(KTR_CRITICAL) || !defined(_KERNEL) || defined(GENOFFSET) +#define critical_enter() critical_enter_KBI() +#define critical_exit() critical_exit_KBI() +#else +static __inline void +critical_enter(void) +{ + struct thread_lite *td; + + td = (struct thread_lite *)curthread; + td->td_critnest++; +} + +static __inline void +critical_exit(void) +{ + struct thread_lite *td; + + td = (struct thread_lite *)curthread; + KASSERT(td->td_critnest != 0, + ("critical_exit: td_critnest == 0")); + td->td_critnest--; + __compiler_membar(); + if (__predict_false(td->td_owepreempt)) + critical_exit_preempt(); + +} +#endif + + #ifdef EARLY_PRINTF typedef void early_putc_t(int ch); extern early_putc_t *early_putc;
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201807030155.w631tARo009470>