Date: Wed, 19 Jul 2023 14:59:51 GMT From: Christos Margiolis <christos@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 5b701ed19c2e - main - kinst: start moving towards per-probe trampolines Message-ID: <202307191459.36JExpHT067585@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by christos: URL: https://cgit.FreeBSD.org/src/commit/?id=5b701ed19c2ed439457b0177681472aac21fde94 commit 5b701ed19c2ed439457b0177681472aac21fde94 Author: Christos Margiolis <christos@FreeBSD.org> AuthorDate: 2023-07-19 14:57:21 +0000 Commit: Christos Margiolis <christos@FreeBSD.org> CommitDate: 2023-07-19 14:57:21 +0000 kinst: start moving towards per-probe trampolines Using per-CPU and per-thread trampolines is expensive and error-prone, since we're rewriting the same memory blocks constantly. Per-probe trampolines solve this problem by giving each probe its own block of executable memory, which more or less remains the same after the initial write. What this patch does, is get rid of the initialization code which allocates a trampoline for each thread, and instead let each port of kinst allocate a trampoline for each new probe created. It also sets up the infrastructure needed to support the new trampoline scheme. This change is not currently supported on amd64, as the amd64 port needs further changes to work, so this is a temporary/gradual patch to fix the riscv and arm64 ports. Reviewed by: markj Approved by: markj (mentor) Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D40962 --- sys/cddl/dev/kinst/kinst.c | 3 +++ sys/cddl/dev/kinst/kinst.h | 28 ++++++++++++++++++++++++++++ sys/cddl/dev/kinst/trampoline.c | 35 ++++++++++++++++++++++++++++++----- 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/sys/cddl/dev/kinst/kinst.c b/sys/cddl/dev/kinst/kinst.c index 4bd3047f49db..60400a452b95 100644 --- a/sys/cddl/dev/kinst/kinst.c +++ b/sys/cddl/dev/kinst/kinst.c @@ -228,6 +228,9 @@ kinst_destroy(void *arg, dtrace_id_t id, void *parg) struct kinst_probe *kp = parg; LIST_REMOVE(kp, kp_hashnext); +#ifndef __amd64__ + kinst_trampoline_dealloc(kp->kp_tramp); +#endif free(kp, M_KINST); } diff --git a/sys/cddl/dev/kinst/kinst.h b/sys/cddl/dev/kinst/kinst.h index 0a47eb4f3583..390a2d1c13bf 100644 --- a/sys/cddl/dev/kinst/kinst.h +++ b/sys/cddl/dev/kinst/kinst.h @@ -35,10 +35,38 @@ struct kinst_probe { kinst_patchval_t kp_patchval; kinst_patchval_t kp_savedval; kinst_patchval_t *kp_patchpoint; + uint8_t *kp_tramp; struct kinst_probe_md kp_md; }; +struct kinst_cpu_state { + /* + * kinst uses a breakpoint to return from the trampoline and resume + * execution. To do this safely, kinst implements a per-CPU state + * machine; the state is set to KINST_PROBE_FIRED for the duration of + * the trampoline execution (i.e from the time we transfer execution to + * it, until we return). Upon return, the state is set to + * KINST_PROBE_ARMED to indicate that a probe is not currently firing. + * All CPUs have their state initialized to KINST_PROBE_ARMED when + * kinst is loaded. + */ + enum { + KINST_PROBE_ARMED, + KINST_PROBE_FIRED, + } state; + /* + * Points to the probe whose trampoline we're currently executing. + */ + struct kinst_probe *kp; + /* + * Because we execute trampolines with interrupts disabled, we have to + * cache the CPU's status in order to restore it when we return from + * the trampoline. + */ + uint64_t status; +}; + LIST_HEAD(kinst_probe_list, kinst_probe); extern struct kinst_probe_list *kinst_probetab; diff --git a/sys/cddl/dev/kinst/trampoline.c b/sys/cddl/dev/kinst/trampoline.c index 87c01e39745b..adc4eaa7fceb 100644 --- a/sys/cddl/dev/kinst/trampoline.c +++ b/sys/cddl/dev/kinst/trampoline.c @@ -49,8 +49,10 @@ static TAILQ_HEAD(, trampchunk) kinst_trampchunks = TAILQ_HEAD_INITIALIZER(kinst_trampchunks); static struct sx kinst_tramp_sx; SX_SYSINIT(kinst_tramp_sx, &kinst_tramp_sx, "kinst tramp"); +#ifdef __amd64__ static eventhandler_tag kinst_thread_ctor_handler; static eventhandler_tag kinst_thread_dtor_handler; +#endif /* * Fill the trampolines with KINST_TRAMP_FILL_PATTERN so that the kernel will @@ -150,12 +152,14 @@ kinst_trampoline_alloc_locked(int how) if ((how & M_NOWAIT) != 0) return (NULL); - /* - * We didn't find any free trampoline in the current list, - * allocate a new one. If that fails the provider will no - * longer be reliable, so try to warn the user. - */ if ((chunk = kinst_trampchunk_alloc()) == NULL) { +#ifdef __amd64__ + /* + * We didn't find any free trampoline in the current + * list, allocate a new one. If that fails the + * provider will no longer be reliable, so try to warn + * the user. + */ static bool once = true; if (once) { @@ -164,6 +168,7 @@ kinst_trampoline_alloc_locked(int how) "kinst: failed to allocate trampoline, " "probes may not fire"); } +#endif return (NULL); } off = 0; @@ -220,6 +225,7 @@ kinst_trampoline_dealloc(uint8_t *tramp) sx_xunlock(&kinst_tramp_sx); } +#ifdef __amd64__ static void kinst_thread_ctor(void *arg __unused, struct thread *td) { @@ -240,10 +246,12 @@ kinst_thread_dtor(void *arg __unused, struct thread *td) */ kinst_trampoline_dealloc(tramp); } +#endif int kinst_trampoline_init(void) { +#ifdef __amd64__ struct proc *p; struct thread *td; void *tramp; @@ -296,12 +304,21 @@ retry: out: sx_xunlock(&kinst_tramp_sx); sx_sunlock(&allproc_lock); +#else + int error = 0; + + sx_xlock(&kinst_tramp_sx); + TAILQ_INIT(&kinst_trampchunks); + sx_xunlock(&kinst_tramp_sx); +#endif + return (error); } int kinst_trampoline_deinit(void) { +#ifdef __amd64__ struct trampchunk *chunk, *tmp; struct proc *p; struct thread *td; @@ -324,6 +341,14 @@ kinst_trampoline_deinit(void) TAILQ_FOREACH_SAFE(chunk, &kinst_trampchunks, next, tmp) kinst_trampchunk_free(chunk); sx_xunlock(&kinst_tramp_sx); +#else + struct trampchunk *chunk, *tmp; + + sx_xlock(&kinst_tramp_sx); + TAILQ_FOREACH_SAFE(chunk, &kinst_trampchunks, next, tmp) + kinst_trampchunk_free(chunk); + sx_xunlock(&kinst_tramp_sx); +#endif return (0); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202307191459.36JExpHT067585>