Date: Tue, 4 Jul 2023 15:39:03 GMT From: Christos Margiolis <christos@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 2d7bb03adb43 - main - kinst: port to riscv Message-ID: <202307041539.364Fd350032693@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by christos: URL: https://cgit.FreeBSD.org/src/commit/?id=2d7bb03adb43dd605464db3a9634ff33bf74ca5f commit 2d7bb03adb43dd605464db3a9634ff33bf74ca5f Author: Christos Margiolis <christos@FreeBSD.org> AuthorDate: 2023-07-04 15:38:01 +0000 Commit: Christos Margiolis <christos@FreeBSD.org> CommitDate: 2023-07-04 15:38:01 +0000 kinst: port to riscv Reviewed by: markj Approved by: markj (mentor) Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D39884 --- sys/cddl/dev/kinst/riscv/kinst_isa.c | 616 +++++++++++++++++++++++++++++++++++ sys/cddl/dev/kinst/riscv/kinst_isa.h | 31 ++ sys/modules/dtrace/Makefile | 3 + 3 files changed, 650 insertions(+) diff --git a/sys/cddl/dev/kinst/riscv/kinst_isa.c b/sys/cddl/dev/kinst/riscv/kinst_isa.c new file mode 100644 index 000000000000..8bd4449a027a --- /dev/null +++ b/sys/cddl/dev/kinst/riscv/kinst_isa.c @@ -0,0 +1,616 @@ +/* + * SPDX-License-Identifier: CDDL 1.0 + * + * Copyright (c) 2023 The FreeBSD Foundation + * + * This software was developed by Christos Margiolis <christos@FreeBSD.org> + * under sponsorship from the FreeBSD Foundation. + */ + +#include <sys/param.h> + +#include <sys/dtrace.h> +#include <cddl/dev/dtrace/dtrace_cddl.h> + +#include "kinst.h" + +/* + * Per-CPU trampolines used when the interrupted thread is executing with + * interrupts disabled. If an interrupt is raised while executing a trampoline, + * the interrupt thread cannot safely overwrite its trampoline if it hits a + * kinst probe while executing the interrupt handler. + */ +DPCPU_DEFINE_STATIC(uint8_t *, intr_tramp); + +/* + * The double-breakpoint mechanism needs to save the current probe for the next + * call to kinst_invop(). As with per-CPU trampolines, this also has to be done + * per-CPU when interrupts are disabled. + */ +DPCPU_DEFINE_STATIC(struct kinst_probe *, intr_probe); + +#define _MATCH_REG(reg) \ + (offsetof(struct trapframe, tf_ ## reg) / sizeof(register_t)) + +static int +kinst_regoff(struct trapframe *frame, int n) +{ + switch (n) { + case 0: + /* There is no zero register in the trapframe structure. */ + return (-1); + case 1: + return (_MATCH_REG(ra)); + case 2: + return (_MATCH_REG(sp)); + case 3: + return (_MATCH_REG(gp)); + case 4: + return (_MATCH_REG(tp)); + case 5 ... 7: + return (_MATCH_REG(t[n - 5])); + case 8 ... 9: + return (_MATCH_REG(s[n - 8])); + case 10 ... 17: + return (_MATCH_REG(a[n - 10])); + case 18 ... 27: + return (_MATCH_REG(s[n - 18 + 2])); + case 28 ... 31: + return (_MATCH_REG(t[n - 28 + 3])); + default: + panic("%s: unhandled register index %d", __func__, n); + } +} + +static int +kinst_c_regoff(struct trapframe *frame, int n) +{ + switch (n) { + case 0 ... 1: + return (_MATCH_REG(s[n])); + case 2 ... 7: + return (_MATCH_REG(a[n - 2])); + default: + panic("%s: unhandled register index %d", __func__, n); + } +} + +#undef _MATCH_REG + +static int +kinst_emulate(struct trapframe *frame, struct kinst_probe *kp) +{ + kinst_patchval_t instr = kp->kp_savedval; + register_t prevpc; + uint64_t imm; + uint16_t off; + uint8_t funct; + + if (kp->kp_md.instlen == INSN_SIZE) { +#define rs1_index ((instr & RS1_MASK) >> RS1_SHIFT) +#define rs2_index ((instr & RS2_MASK) >> RS2_SHIFT) +#define rd_index ((instr & RD_MASK) >> RD_SHIFT) +#define rs1 ((register_t *)frame)[kinst_regoff(frame, rs1_index)] +#define rs2 ((register_t *)frame)[kinst_regoff(frame, rs2_index)] +#define rd ((register_t *)frame)[kinst_regoff(frame, rd_index)] +#define rs1_lval (rs1_index != 0 ? rs1 : 0) +#define rs2_lval (rs2_index != 0 ? rs2 : 0) + switch (instr & 0x7f) { + case 0b1101111: /* jal */ + imm = 0; + imm |= ((instr >> 21) & 0x03ff) << 1; + imm |= ((instr >> 20) & 0x0001) << 11; + imm |= ((instr >> 12) & 0x00ff) << 12; + imm |= ((instr >> 31) & 0x0001) << 20; + if (imm & 0x0000000000100000) + imm |= 0xfffffffffff00000; + if (rd_index != 0) + rd = frame->tf_sepc + INSN_SIZE; + frame->tf_sepc += imm; + break; + case 0b1100111: /* jalr */ + prevpc = frame->tf_sepc; + imm = (instr & IMM_MASK) >> IMM_SHIFT; + if (imm & 0x0000000000000800) + imm |= 0xfffffffffffff000; + frame->tf_sepc = (rs1_lval + imm) & ~1; + if (rd_index != 0) + rd = prevpc + INSN_SIZE; + break; + case 0b1100011: /* branch */ + imm = 0; + imm |= ((instr >> 8) & 0x000f) << 1; + imm |= ((instr >> 25) & 0x003f) << 5; + imm |= ((instr >> 7) & 0x0001) << 11; + imm |= ((instr >> 31) & 0x0001) << 12; + if (imm & 0x0000000000001000) + imm |= 0xfffffffffffff000; + funct = (instr >> 12) & 0x07; + switch (funct) { + case 0b000: /* beq */ + if (rs1_lval == rs2_lval) + frame->tf_sepc += imm; + else + frame->tf_sepc += INSN_SIZE; + break; + case 0b001: /* bne */ + if (rs1_lval != rs2_lval) + frame->tf_sepc += imm; + else + frame->tf_sepc += INSN_SIZE; + break; + case 0b100: /* blt */ + if ((int64_t)rs1_lval < (int64_t)rs2_lval) + frame->tf_sepc += imm; + else + frame->tf_sepc += INSN_SIZE; + break; + case 0b110: /* bltu */ + if ((uint64_t)rs1_lval < (uint64_t)rs2_lval) + frame->tf_sepc += imm; + else + frame->tf_sepc += INSN_SIZE; + break; + case 0b101: /* bge */ + if ((int64_t)rs1_lval >= (int64_t)rs2_lval) + frame->tf_sepc += imm; + else + frame->tf_sepc += INSN_SIZE; + break; + case 0b111: /* bgeu */ + if ((uint64_t)rs1_lval >= (uint64_t)rs2_lval) + frame->tf_sepc += imm; + else + frame->tf_sepc += INSN_SIZE; + break; + } + break; + case 0b0010111: /* auipc */ + imm = instr & 0xfffff000; + rd = frame->tf_sepc + + (imm & 0x0000000080000000 ? + imm | 0xffffffff80000000 : imm); + frame->tf_sepc += INSN_SIZE; + break; + } +#undef rs1_lval +#undef rs2_lval +#undef rs1 +#undef rs2 +#undef rd +#undef rs1_index +#undef rs2_index +#undef rd_index + } else { + switch (instr & 0x03) { +#define rs1 \ + ((register_t *)frame)[kinst_c_regoff(frame, (instr >> 7) & 0x07)] + case 0b01: + funct = (instr >> 13) & 0x07; + switch (funct) { + case 0b101: /* c.j */ + off = (instr >> 2) & 0x07ff; + imm = 0; + imm |= ((off >> 1) & 0x07) << 1; + imm |= ((off >> 9) & 0x01) << 4; + imm |= ((off >> 0) & 0x01) << 5; + imm |= ((off >> 5) & 0x01) << 6; + imm |= ((off >> 4) & 0x01) << 7; + imm |= ((off >> 7) & 0x03) << 8; + imm |= ((off >> 6) & 0x01) << 10; + imm |= ((off >> 10) & 0x01) << 11; + if (imm & 0x0000000000000800) + imm |= 0xfffffffffffff000; + frame->tf_sepc += imm; + break; + case 0b110: /* c.beqz */ + case 0b111: /* c.bnez */ + imm = 0; + imm |= ((instr >> 3) & 0x03) << 1; + imm |= ((instr >> 10) & 0x03) << 3; + imm |= ((instr >> 2) & 0x01) << 5; + imm |= ((instr >> 5) & 0x03) << 6; + imm |= ((instr >> 12) & 0x01) << 8; + if (imm & 0x0000000000000100) + imm |= 0xffffffffffffff00; + if (funct == 0b110 && rs1 == 0) + frame->tf_sepc += imm; + else if (funct == 0b111 && rs1 != 0) + frame->tf_sepc += imm; + else + frame->tf_sepc += INSN_C_SIZE; + break; + } + break; +#undef rs1 +#define rs1_index ((instr & RD_MASK) >> RD_SHIFT) +#define rs1 ((register_t *)frame)[kinst_regoff(frame, rs1_index)] + case 0b10: + funct = (instr >> 13) & 0x07; + if (funct == 0b100 && rs1_index != 0) { + /* c.jr/c.jalr */ + prevpc = frame->tf_sepc; + frame->tf_sepc = rs1; + if (((instr >> 12) & 0x01) != 0) + frame->tf_ra = prevpc + INSN_C_SIZE; + } + break; +#undef rs1 +#undef rs1_index + } + } + + return (MATCH_C_NOP); +} + +static int +kinst_jump_next_instr(struct trapframe *frame, struct kinst_probe *kp) +{ + frame->tf_sepc = (register_t)((uint8_t *)kp->kp_patchpoint + + kp->kp_md.instlen); + + return (MATCH_C_NOP); +} + +static void +kinst_trampoline_populate(struct kinst_probe *kp, uint8_t *tramp) +{ + static uint16_t nop = MATCH_C_NOP; + static uint32_t ebreak = MATCH_EBREAK; + int ilen; + + ilen = kp->kp_md.instlen; + kinst_memcpy(tramp, &kp->kp_savedval, ilen); + + /* + * Since we cannot encode large displacements in a single instruction + * in order to encode a far-jump back to the next instruction, and we + * also cannot clobber a register inside the trampoline, we execute a + * breakpoint after the copied instruction. kinst_invop() is + * responsible for detecting this special case and performing the + * "jump" manually. + * + * Add a NOP after a compressed instruction for padding. + */ + if (ilen == INSN_C_SIZE) + kinst_memcpy(&tramp[ilen], &nop, INSN_C_SIZE); + + kinst_memcpy(&tramp[INSN_SIZE], &ebreak, INSN_SIZE); + + fence_i(); +} + +/* + * There are two ways by which an instruction is traced: + * + * - By using the trampoline. + * - By emulating it in software (see kinst_emulate()). + * + * The trampoline is used for instructions that can be copied and executed + * as-is without additional modification. However, instructions that use + * PC-relative addressing have to be emulated, because RISC-V doesn't allow + * encoding of large displacements in a single instruction, and since we cannot + * clobber a register in order to encode the two-instruction sequence needed to + * create large displacements, we cannot use the trampoline at all. + * Fortunately, the instructions are simple enough to be emulated in just a few + * lines of code. + * + * The problem discussed above also means that, unlike amd64, we cannot encode + * a far-jump back from the trampoline to the next instruction. The mechanism + * employed to achieve this functionality, is to use a breakpoint instead of a + * jump after the copied instruction. This breakpoint is detected and handled + * by kinst_invop(), which performs the jump back to the next instruction + * manually (see kinst_jump_next_instr()). + */ +int +kinst_invop(uintptr_t addr, struct trapframe *frame, uintptr_t scratch) +{ + solaris_cpu_t *cpu; + struct kinst_probe *kp; + uint8_t *tramp; + + /* + * Use per-CPU trampolines and probes if the thread executing the + * instruction was executing with interrupts disabled. + */ + if ((frame->tf_sstatus & SSTATUS_SPIE) == 0) { + tramp = DPCPU_GET(intr_tramp); + kp = DPCPU_GET(intr_probe); + } else { + tramp = curthread->t_kinst_tramp; + kp = curthread->t_kinst_curprobe; + } + + /* + * Detect if the breakpoint was triggered by the trampoline, and + * manually set the PC to the next instruction. + */ + if (addr == (uintptr_t)(tramp + INSN_SIZE)) + return (kinst_jump_next_instr(frame, kp)); + + LIST_FOREACH(kp, KINST_GETPROBE(addr), kp_hashnext) { + if ((uintptr_t)kp->kp_patchpoint == addr) + break; + } + if (kp == NULL) + return (0); + + cpu = &solaris_cpu[curcpu]; + cpu->cpu_dtrace_caller = addr; + dtrace_probe(kp->kp_id, 0, 0, 0, 0, 0); + cpu->cpu_dtrace_caller = 0; + + if (kp->kp_md.emulate) + return (kinst_emulate(frame, kp)); + + if (tramp == NULL) { + /* + * A trampoline allocation failed, so this probe is + * effectively disabled. Restore the original + * instruction. + * + * We can't safely print anything here, but the + * trampoline allocator should have left a breadcrumb in + * the dmesg. + */ + kinst_patch_tracepoint(kp, kp->kp_savedval); + frame->tf_sepc = (register_t)kp->kp_patchpoint; + } else { + kinst_trampoline_populate(kp, tramp); + frame->tf_sepc = (register_t)tramp; + if ((frame->tf_sstatus & SSTATUS_SPIE) == 0) + DPCPU_SET(intr_probe, kp); + else + curthread->t_kinst_curprobe = kp; + } + + return (MATCH_C_NOP); +} + +void +kinst_patch_tracepoint(struct kinst_probe *kp, kinst_patchval_t val) +{ + switch (kp->kp_patchval) { + case KINST_C_PATCHVAL: + *(uint16_t *)kp->kp_patchpoint = (uint16_t)val; + fence_i(); + break; + case KINST_PATCHVAL: + *kp->kp_patchpoint = val; + fence_i(); + break; + } +} + +static void +kinst_instr_dissect(struct kinst_probe *kp, int instrsize) +{ + struct kinst_probe_md *kpmd; + kinst_patchval_t instr = kp->kp_savedval; + uint8_t funct; + + kpmd = &kp->kp_md; + kpmd->instlen = instrsize; + kpmd->emulate = false; + + /* + * The following instructions use PC-relative addressing and need to be + * emulated in software. + */ + if (kpmd->instlen == INSN_SIZE) { + switch (instr & 0x7f) { + case 0b1101111: /* jal */ + case 0b1100111: /* jalr */ + case 0b1100011: /* branch */ + case 0b0010111: /* auipc */ + kpmd->emulate = true; + break; + } + } else { + switch (instr & 0x03) { + case 0b01: + funct = (instr >> 13) & 0x07; + switch (funct) { + case 0b101: /* c.j */ + case 0b110: /* c.beqz */ + case 0b111: /* c.bnez */ + kpmd->emulate = true; + break; + } + break; + case 0b10: + funct = (instr >> 13) & 0x07; + if (funct == 0b100 && + ((instr >> 7) & 0x1f) != 0 && + ((instr >> 2) & 0x1f) == 0) + kpmd->emulate = true; /* c.jr/c.jalr */ + break; + } + } +} + +static bool +kinst_instr_system(kinst_patchval_t instr) +{ + if (dtrace_match_opcode(instr, MATCH_C_EBREAK, MASK_C_EBREAK) || + (instr & 0x7f) == 0b1110011) + return (true); + + return (false); +} + +static bool +kinst_instr_lr(kinst_patchval_t instr) +{ + if (dtrace_match_opcode(instr, MATCH_LR_W, MASK_LR_W) || + dtrace_match_opcode(instr, MATCH_LR_D, MASK_LR_D)) + return (true); + + return (false); +} + +static bool +kinst_instr_sc(kinst_patchval_t instr) +{ + if (dtrace_match_opcode(instr, MATCH_SC_W, MASK_SC_W) || + dtrace_match_opcode(instr, MATCH_SC_D, MASK_SC_D)) + return (true); + + return (false); +} + +int +kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval, + void *opaque) +{ + struct kinst_probe *kp; + dtrace_kinst_probedesc_t *pd; + const char *func; + kinst_patchval_t *insn, v; + uint8_t *instr, *limit; + int instrsize, n, off; + bool lrsc_block, store_found, ret_found; + + pd = opaque; + func = symval->name; + + if (kinst_excluded(func)) + return (0); + if (strcmp(func, pd->kpd_func) != 0) + return (0); + + instr = (uint8_t *)(symval->value); + limit = (uint8_t *)(symval->value + symval->size); + if (instr >= limit) + return (0); + + /* Check for the usual function prologue. */ + for (insn = (kinst_patchval_t *)instr; + insn < (kinst_patchval_t *)limit; insn++) { + if (dtrace_instr_sdsp(&insn) || dtrace_instr_c_sdsp(&insn)) + store_found = true; + else if (dtrace_instr_ret(&insn) || dtrace_instr_c_ret(&insn)) + ret_found = true; + if (store_found && ret_found) + break; + } + if (!store_found || !ret_found) + return (0); + + n = 0; + lrsc_block = false; + while (instr < limit) { + instrsize = dtrace_instr_size(instr); + off = (int)(instr - (uint8_t *)symval->value); + + /* + * Avoid undefined behavior (i.e simply casting `*instr` to + * `kinst_patchval_t`) in case the pointer is unaligned. + * memcpy() can safely operate on unaligned pointers. + */ + memcpy(&v, instr, sizeof(kinst_patchval_t)); + + /* Skip SYSTEM instructions. */ + if (kinst_instr_system(v)) + goto cont; + + /* + * Skip LR/SC blocks used to build atomic operations. If a + * breakpoint is placed in a LR/SC block, the loop becomes + * unconstrained. In this case we violate the operation and the + * loop might fail on some implementations (see section 8.3 of + * the RISC-V unprivileged spec). + */ + if (kinst_instr_lr(v)) + lrsc_block = true; + else if (kinst_instr_sc(v)) { + lrsc_block = false; + goto cont; + } + if (lrsc_block) + goto cont; + + if (pd->kpd_off != -1 && off != pd->kpd_off) + goto cont; + + /* + * Prevent separate dtrace(1) instances from creating copies of + * the same probe. + */ + LIST_FOREACH(kp, KINST_GETPROBE(instr), kp_hashnext) { + if (strcmp(kp->kp_func, func) == 0 && + strtol(kp->kp_name, NULL, 10) == off) + return (0); + } + if (++n > KINST_PROBETAB_MAX) { + KINST_LOG("probe list full: %d entries", n); + return (ENOMEM); + } + kp = malloc(sizeof(struct kinst_probe), M_KINST, + M_WAITOK | M_ZERO); + kp->kp_func = func; + snprintf(kp->kp_name, sizeof(kp->kp_name), "%d", off); + kp->kp_patchpoint = (kinst_patchval_t *)instr; + kp->kp_savedval = v; + if (instrsize == INSN_SIZE) + kp->kp_patchval = KINST_PATCHVAL; + else + kp->kp_patchval = KINST_C_PATCHVAL; + + kinst_instr_dissect(kp, instrsize); + kinst_probe_create(kp, lf); +cont: + instr += instrsize; + } + if (lrsc_block) + KINST_LOG("warning: unterminated LR/SC block"); + + return (0); +} + +int +kinst_md_init(void) +{ + uint8_t *tramp; + int cpu; + + CPU_FOREACH(cpu) { + tramp = kinst_trampoline_alloc(M_WAITOK); + if (tramp == NULL) + return (ENOMEM); + DPCPU_ID_SET(cpu, intr_tramp, tramp); + } + + return (0); +} + +void +kinst_md_deinit(void) +{ + uint8_t *tramp; + int cpu; + + CPU_FOREACH(cpu) { + tramp = DPCPU_ID_GET(cpu, intr_tramp); + if (tramp != NULL) { + kinst_trampoline_dealloc(tramp); + DPCPU_ID_SET(cpu, intr_tramp, NULL); + } + } +} + +/* + * Exclude machine-dependent functions that are not safe-to-trace. + */ +bool +kinst_md_excluded(const char *name) +{ + if (strcmp(name, "cpu_exception_handler") == 0 || + strcmp(name, "cpu_exception_handler_supervisor") == 0 || + strcmp(name, "cpu_exception_handler_user") == 0 || + strcmp(name, "do_trap_supervisor") == 0 || + strcmp(name, "do_trap_user") == 0) + return (true); + + return (false); +} diff --git a/sys/cddl/dev/kinst/riscv/kinst_isa.h b/sys/cddl/dev/kinst/riscv/kinst_isa.h new file mode 100644 index 000000000000..bd7c2b3b8dcf --- /dev/null +++ b/sys/cddl/dev/kinst/riscv/kinst_isa.h @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: CDDL 1.0 + * + * Copyright (c) 2023 The FreeBSD Foundation + * + * This software was developed by Christos Margiolis <christos@FreeBSD.org> + * under sponsorship from the FreeBSD Foundation. + */ + +#ifndef _KINST_ISA_H_ +#define _KINST_ISA_H_ + +#include <machine/riscvreg.h> +#include <machine/encoding.h> + +#define KINST_PATCHVAL MATCH_EBREAK +#define KINST_C_PATCHVAL MATCH_C_EBREAK + +/* + * The trampoline contains [instruction, [nop padding], ebreak]. + */ +#define KINST_TRAMP_SIZE 8 + +typedef uint32_t kinst_patchval_t; + +struct kinst_probe_md { + int instlen; /* original instr len */ + bool emulate; /* emulate in sw */ +}; + +#endif /* _KINST_ISA_H_ */ diff --git a/sys/modules/dtrace/Makefile b/sys/modules/dtrace/Makefile index 31beadb4d890..e38e2687e8a7 100644 --- a/sys/modules/dtrace/Makefile +++ b/sys/modules/dtrace/Makefile @@ -21,6 +21,9 @@ SUBDIR+= fasttrap systrace_linux SUBDIR+= systrace_linux32 SUBDIR+= kinst .endif +.if ${MACHINE_CPUARCH} == "riscv" +SUBDIR+= kinst +.endif .if ${MACHINE_CPUARCH} == "powerpc" SUBDIR+= fasttrap .endif
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202307041539.364Fd350032693>