Date: Fri, 1 Feb 2008 03:51:49 GMT From: John Birrell <jb@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 134576 for review Message-ID: <200802010351.m113pnm6046332@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=134576 Change 134576 by jb@jb_freebsd1 on 2008/02/01 03:50:58 IFdtrace. Exception handling for fbt. Timer callback for cyclic. Trap handing for safety. And syscall probe mumbo. Affected files ... .. //depot/projects/dtrace7/src/sys/amd64/amd64/exception.S#3 edit .. //depot/projects/dtrace7/src/sys/amd64/amd64/local_apic.c#2 edit .. //depot/projects/dtrace7/src/sys/amd64/amd64/trap.c#4 edit Differences ... ==== //depot/projects/dtrace7/src/sys/amd64/amd64/exception.S#3 (text+ko) ==== @@ -1,8 +1,12 @@ /*- * Copyright (c) 1989, 1990 William F. Jolitz. * Copyright (c) 1990 The Regents of the University of California. + * Copyright (c) 2007 The FreeBSD Foundation * All rights reserved. * + * Portions of this software were developed by A. Joseph Koshy under + * sponsorship from the FreeBSD Foundation and Google, Inc. + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: @@ -27,11 +31,13 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $FreeBSD: src/sys/amd64/amd64/exception.S,v 1.129.2.1 2007/11/21 16:38:54 jhb Exp $ + * $FreeBSD: src/sys/amd64/amd64/exception.S,v 1.131 2007/12/07 08:20:15 jkoshy Exp $ */ #include "opt_atpic.h" #include "opt_compat.h" +#include "opt_hwpmc_hooks.h" +#include "opt_kdtrace.h" #include <machine/asmacros.h> #include <machine/psl.h> @@ -39,7 +45,25 @@ #include "assym.s" +#ifdef KDTRACE_HOOKS + .bss + .globl dtrace_invop_jump_addr + .align 8 + .type dtrace_invop_jump_addr, @object + .size dtrace_invop_jump_addr, 8 +dtrace_invop_jump_addr: + .zero 8 + .globl dtrace_invop_calltrap_addr + .align 8 + .type dtrace_invop_calltrap_addr, @object + .size dtrace_invop_calltrap_addr, 8 +dtrace_invop_calltrap_addr: + .zero 8 +#endif .text +#ifdef HWPMC_HOOKS + ENTRY(start_exceptions) +#endif /*****************************************************************************/ /* Trap handling */ @@ -162,6 +186,30 @@ movq %r14,TF_R14(%rsp) movq %r15,TF_R15(%rsp) FAKE_MCOUNT(TF_RIP(%rsp)) +#ifdef KDTRACE_HOOKS + /* + * DTrace Function Boundary Trace (fbt) and Statically Defined + * Trace (sdt) probes are triggered by int3 (0xcc) which causes + * the #BP (T_BPTFLT) breakpoint interrupt. For all other trap + * types, just handle them in the usual way. + */ + cmpq $T_BPTFLT,TF_TRAPNO(%rsp) + jne calltrap + + /* Check if there is no DTrace hook registered. */ + cmpq $0,dtrace_invop_jump_addr + je calltrap + + /* + * Set our jump address for the jump back in the event that + * the breakpoint wasn't caused by DTrace at all. + */ + movq $calltrap, dtrace_invop_calltrap_addr(%rip) + + /* Jump to the code hooked in by DTrace. */ + movq dtrace_invop_jump_addr, %rax + jmpq *dtrace_invop_jump_addr +#endif .globl calltrap .type calltrap,@function calltrap: @@ -348,6 +396,9 @@ * execute the NMI handler with interrupts disabled to prevent a * nested interrupt from executing an 'iretq' instruction and * inadvertently taking the processor out of NMI mode. + * + * Third, the NMI handler runs on its own stack (tss_ist1), shared + * with the double fault handler. */ IDTVEC(nmi) @@ -386,6 +437,61 @@ movq %rsp, %rdi call trap MEXITCOUNT +#ifdef HWPMC_HOOKS + /* + * Check if the current trap was from user mode and if so + * whether the current thread needs a user call chain to be + * captured. We are still in NMI mode at this point. + */ + testb $SEL_RPL_MASK,TF_CS(%rsp) + jz nocallchain + movq PCPU(CURTHREAD),%rax /* curthread present? */ + orq %rax,%rax + jz nocallchain + testl $TDP_CALLCHAIN,TD_PFLAGS(%rax) /* flagged for capture? */ + jz nocallchain + /* + * A user callchain is to be captured, so: + * - Move execution to the regular kernel stack, to allow for + * nested NMI interrupts. + * - Take the processor out of "NMI" mode by faking an "iret". + * - Enable interrupts, so that copyin() can work. + */ + movq %rsp,%rsi /* source stack pointer */ + movq $TF_SIZE,%rcx + movq PCPU(RSP0),%rbx + subq %rcx,%rbx + movq %rbx,%rdi /* destination stack pointer */ + + shrq $3,%rcx /* trap frame size in long words */ + cld + rep + movsq /* copy trapframe */ + + movl %ss,%eax + pushq %rax /* tf_ss */ + pushq %rbx /* tf_rsp (on kernel stack) */ + pushfq /* tf_rflags */ + movl %cs,%eax + pushq %rax /* tf_cs */ + pushq $outofnmi /* tf_rip */ + iretq +outofnmi: + /* + * At this point the processor has exited NMI mode and is running + * with interrupts turned off on the normal kernel stack. + * We turn interrupts back on, and take the usual 'doreti' exit + * path. + * + * If a pending NMI gets recognized at or after this point, it + * will cause a kernel callchain to be traced. Since this path + * is only taken for NMI interrupts from user space, our `swapgs' + * state is correct for taking the doreti path. + */ + sti + jmp doreti +nocallchain: +#endif testl %ebx,%ebx jz nmi_restoreregs swapgs @@ -556,3 +662,6 @@ movq $0,TF_ADDR(%rsp) FAKE_MCOUNT(TF_RIP(%rsp)) jmp calltrap +#ifdef HWPMC_HOOKS + ENTRY(end_exceptions) +#endif ==== //depot/projects/dtrace7/src/sys/amd64/amd64/local_apic.c#2 (text+ko) ==== @@ -32,9 +32,10 @@ */ #include <sys/cdefs.h> -__FBSDID("$FreeBSD: src/sys/amd64/amd64/local_apic.c,v 1.42.2.1 2007/11/08 20:09:15 jhb Exp $"); +__FBSDID("$FreeBSD: src/sys/amd64/amd64/local_apic.c,v 1.43 2007/10/27 13:34:53 jhb Exp $"); #include "opt_hwpmc_hooks.h" +#include "opt_kdtrace.h" #include "opt_ddb.h" @@ -65,6 +66,11 @@ #include <ddb/ddb.h> #endif +#ifdef KDTRACE_HOOKS +#include <sys/dtrace_bsd.h> +cyclic_clock_func_t lapic_cyclic_clock_func[MAXCPU]; +#endif + /* Sanity checks on IDT vectors. */ CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT); CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS); @@ -668,6 +674,17 @@ (*la->la_timer_count)++; critical_enter(); +#ifdef KDTRACE_HOOKS + /* + * If the DTrace hooks are configured and a callback function + * has been registered, then call it to process the high speed + * timers. + */ + int cpu = PCPU_GET(cpuid); + if (lapic_cyclic_clock_func[cpu] != NULL) + (*lapic_cyclic_clock_func[cpu])(frame); +#endif + /* Fire hardclock at hz. */ la->la_hard_ticks += hz; if (la->la_hard_ticks >= lapic_timer_hz) { ==== //depot/projects/dtrace7/src/sys/amd64/amd64/trap.c#4 (text+ko) ==== @@ -38,7 +38,7 @@ */ #include <sys/cdefs.h> -__FBSDID("$FreeBSD: src/sys/amd64/amd64/trap.c,v 1.319.2.2 2007/12/06 14:20:25 kib Exp $"); +__FBSDID("$FreeBSD: src/sys/amd64/amd64/trap.c,v 1.324 2007/12/07 08:20:15 jkoshy Exp $"); /* * AMD64 Trap and System call handling @@ -49,6 +49,7 @@ #include "opt_hwpmc_hooks.h" #include "opt_isa.h" #include "opt_kdb.h" +#include "opt_kdtrace.h" #include "opt_ktrace.h" #include <sys/param.h> @@ -94,6 +95,24 @@ #endif #include <machine/tss.h> +#ifdef KDTRACE_HOOKS +#include <sys/dtrace_bsd.h> + +/* + * This is a hook which is initialised by the dtrace module + * to handle traps which might occur during DTrace probe + * execution. + */ +dtrace_trap_func_t dtrace_trap_func; + +/* + * This is a hook which is initialised by the systrace module + * when it is loaded. This keeps the DTrace syscall provider + * implementation opaque. + */ +systrace_probe_func_t systrace_probe_func; +#endif + extern void trap(struct trapframe *frame); extern void syscall(struct trapframe *frame); void dblfault_handler(struct trapframe *frame); @@ -195,11 +214,28 @@ * the NMI was handled by it and we can return immediately. */ if (type == T_NMI && pmc_intr && - (*pmc_intr)(PCPU_GET(cpuid), (uintptr_t) frame->tf_rip, - TRAPF_USERMODE(frame))) + (*pmc_intr)(PCPU_GET(cpuid), frame)) goto out; #endif +#ifdef KDTRACE_HOOKS + /* + * A trap can occur while DTrace executes a probe. Before + * executing the probe, DTrace blocks re-scheduling and sets + * a flag in it's per-cpu flags to indicate that it doesn't + * want to fault. On returning from the the probe, the no-fault + * flag is cleared and finally re-scheduling is enabled. + * + * If the DTrace kernel module has registered a trap handler, + * call it and if it returns non-zero, assume that it has + * handled the trap and modified the trap frame so that this + * function can return normally. + */ + if (dtrace_trap_func != NULL) + if ((*dtrace_trap_func)(frame, type)) + goto out; +#endif + if ((frame->tf_rflags & PSL_I) == 0) { /* * Buggy application or kernel code has disabled @@ -211,7 +247,7 @@ if (ISPL(frame->tf_cs) == SEL_UPL) printf( "pid %ld (%s): trap %d with interrupts disabled\n", - (long)curproc->p_pid, curproc->p_comm, type); + (long)curproc->p_pid, curthread->td_name, type); else if (type != T_NMI && type != T_BPTFLT && type != T_TRCTRAP) { /* @@ -708,8 +744,8 @@ printf("current process = "); if (curproc) { printf("%lu (%s)\n", - (u_long)curproc->p_pid, curproc->p_comm ? - curproc->p_comm : ""); + (u_long)curproc->p_pid, curthread->td_name ? + curthread->td_name : ""); } else { printf("Idle\n"); } @@ -836,7 +872,7 @@ #endif CTR4(KTR_SYSC, "syscall enter thread %p pid %d proc %s code %d", td, - td->td_proc->p_pid, td->td_proc->p_comm, code); + td->td_proc->p_pid, td->td_name, code); td->td_syscalls++; @@ -848,9 +884,38 @@ PTRACESTOP_SC(p, td, S_PT_SCE); +#ifdef KDTRACE_HOOKS + /* + * If the systrace module has registered it's probe + * callback and if there is a probe active for the + * syscall 'entry', process the probe. + */ + if (systrace_probe_func != NULL && callp->sy_entry != 0) + (*systrace_probe_func)(callp->sy_entry, code, callp, + args); +#endif + AUDIT_SYSCALL_ENTER(code, td); error = (*callp->sy_call)(td, argp); AUDIT_SYSCALL_EXIT(error, td); + + /* Save the latest error return value. */ + td->td_errno = error; + +#ifdef KDTRACE_HOOKS + /* Save the error return variable for DTrace to reference. */ + args[0] = error; + args[1] = error; + + /* + * If the systrace module has registered it's probe + * callback and if there is a probe active for the + * syscall 'return', process the probe. + */ + if (systrace_probe_func != NULL && callp->sy_return != 0) + (*systrace_probe_func)(callp->sy_return, code, callp, + args); +#endif } switch (error) { @@ -918,7 +983,7 @@ userret(td, frame); CTR4(KTR_SYSC, "syscall exit thread %p pid %d proc %s code %d", td, - td->td_proc->p_pid, td->td_proc->p_comm, code); + td->td_proc->p_pid, td->td_name, code); #ifdef KTRACE if (KTRPOINT(td, KTR_SYSRET))
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200802010351.m113pnm6046332>