Date: Thu, 29 Sep 2011 00:39:57 +0000 (UTC) From: Konstantin Belousov <kib@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-8@freebsd.org Subject: svn commit: r225855 - in stable/8/sys: amd64/amd64 amd64/ia32 amd64/include i386/i386 ia64/ia32 ia64/ia64 kern powerpc/aim powerpc/booke sparc64/sparc64 sun4v/sun4v sys Message-ID: <201109290039.p8T0dvEo096785@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kib Date: Thu Sep 29 00:39:56 2011 New Revision: 225855 URL: http://svn.freebsd.org/changeset/base/225855 Log: Merge the optimizations for the syscall entry and leave. MFC r225474: Inline the syscallenter() and syscallret(). This reduces the time measured by the syscall entry speed microbenchmarks by ~10% on amd64. MFC r225475: Perform amd64-specific microoptimizations for native syscall entry sequence. The effect is ~1% on the microbenchmark. In particular, do not restore registers which are preserved by the C calling sequence. Align the jump target. Avoid unneeded memory accesses by calculating some data in syscall entry trampoline. MFC r225483: The jump target shall be after the padding, not into it. MFC r225575: Microoptimize the return path for the fast syscalls on amd64. Arrange the code to have the fall-through path to follow the likely target. Do not use intermediate register to reload user %rsp. MFC r225576: Put amd64_syscall() prototype in md_var.h. Tested by: Alexandr Kovalenko <never nevermind kiev ua> Added: stable/8/sys/kern/subr_syscall.c - copied, changed from r225474, head/sys/kern/subr_syscall.c Modified: stable/8/sys/amd64/amd64/exception.S stable/8/sys/amd64/amd64/genassym.c stable/8/sys/amd64/amd64/trap.c stable/8/sys/amd64/ia32/ia32_syscall.c stable/8/sys/amd64/include/md_var.h stable/8/sys/i386/i386/trap.c stable/8/sys/ia64/ia32/ia32_trap.c stable/8/sys/ia64/ia64/trap.c stable/8/sys/kern/subr_trap.c stable/8/sys/powerpc/aim/trap.c stable/8/sys/powerpc/booke/trap.c stable/8/sys/sparc64/sparc64/trap.c stable/8/sys/sun4v/sun4v/trap.c stable/8/sys/sys/proc.h Directory Properties: stable/8/sys/ (props changed) stable/8/sys/amd64/include/xen/ (props changed) stable/8/sys/cddl/contrib/opensolaris/ (props changed) stable/8/sys/contrib/dev/acpica/ (props changed) stable/8/sys/contrib/pf/ (props changed) Modified: stable/8/sys/amd64/amd64/exception.S ============================================================================== --- stable/8/sys/amd64/amd64/exception.S Wed Sep 28 21:42:13 2011 (r225854) +++ stable/8/sys/amd64/amd64/exception.S Thu Sep 29 00:39:56 2011 (r225855) @@ -380,8 +380,11 @@ IDTVEC(fast_syscall) movl $TF_HASSEGS,TF_FLAGS(%rsp) cld FAKE_MCOUNT(TF_RIP(%rsp)) - movq %rsp,%rdi - call syscall + movq PCPU(CURTHREAD),%rdi + movq %rsp,TD_FRAME(%rdi) + movl TF_RFLAGS(%rsp),%esi + andl $PSL_T,%esi + call amd64_syscall 1: movq PCPU(CURPCB),%rax /* Disable interrupts before testing PCB_FULL_IRET. */ cli @@ -390,29 +393,25 @@ IDTVEC(fast_syscall) /* Check for and handle AST's on return to userland. */ movq PCPU(CURTHREAD),%rax testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) - je 2f - sti - movq %rsp, %rdi - call ast - jmp 1b -2: /* Restore preserved registers. */ + jne 2f + /* Restore preserved registers. */ MEXITCOUNT movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ movq TF_RDX(%rsp),%rdx /* return value 2 */ movq TF_RAX(%rsp),%rax /* return value 1 */ - movq TF_RBX(%rsp),%rbx /* C preserved */ - movq TF_RBP(%rsp),%rbp /* C preserved */ - movq TF_R12(%rsp),%r12 /* C preserved */ - movq TF_R13(%rsp),%r13 /* C preserved */ - movq TF_R14(%rsp),%r14 /* C preserved */ - movq TF_R15(%rsp),%r15 /* C preserved */ movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ movq TF_RIP(%rsp),%rcx /* original %rip */ - movq TF_RSP(%rsp),%r9 /* user stack pointer */ - movq %r9,%rsp /* original %rsp */ + movq TF_RSP(%rsp),%rsp /* user stack pointer */ swapgs sysretq + +2: /* AST scheduled. */ + sti + movq %rsp,%rdi + call ast + jmp 1b + 3: /* Requested full context restore, use doreti for that. */ MEXITCOUNT jmp doreti Modified: stable/8/sys/amd64/amd64/genassym.c ============================================================================== --- stable/8/sys/amd64/amd64/genassym.c Wed Sep 28 21:42:13 2011 (r225854) +++ stable/8/sys/amd64/amd64/genassym.c Thu Sep 29 00:39:56 2011 (r225855) @@ -87,6 +87,7 @@ ASSYM(TD_PCB, offsetof(struct thread, td ASSYM(TD_PFLAGS, offsetof(struct thread, td_pflags)); ASSYM(TD_PROC, offsetof(struct thread, td_proc)); ASSYM(TD_TID, offsetof(struct thread, td_tid)); +ASSYM(TD_FRAME, offsetof(struct thread, td_frame)); ASSYM(TDF_ASTPENDING, TDF_ASTPENDING); ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED); Modified: stable/8/sys/amd64/amd64/trap.c ============================================================================== --- stable/8/sys/amd64/amd64/trap.c Wed Sep 28 21:42:13 2011 (r225854) +++ stable/8/sys/amd64/amd64/trap.c Thu Sep 29 00:39:56 2011 (r225855) @@ -889,41 +889,37 @@ cpu_fetch_syscall_args(struct thread *td return (error); } +#include "../../kern/subr_syscall.c" + /* * syscall - system call request C handler * * A system call is essentially treated as a trap. */ void -syscall(struct trapframe *frame) +amd64_syscall(struct thread *td, int traced) { - struct thread *td; struct syscall_args sa; - register_t orig_tf_rflags; int error; ksiginfo_t ksi; #ifdef DIAGNOSTIC - if (ISPL(frame->tf_cs) != SEL_UPL) { + if (ISPL(td->td_frame->tf_cs) != SEL_UPL) { panic("syscall"); /* NOT REACHED */ } #endif - orig_tf_rflags = frame->tf_rflags; - td = curthread; - td->td_frame = frame; - error = syscallenter(td, &sa); /* * Traced syscall. */ - if (orig_tf_rflags & PSL_T) { - frame->tf_rflags &= ~PSL_T; + if (__predict_false(traced)) { + td->td_frame->tf_rflags &= ~PSL_T; ksiginfo_init_trap(&ksi); ksi.ksi_signo = SIGTRAP; ksi.ksi_code = TRAP_TRACE; - ksi.ksi_addr = (void *)frame->tf_rip; + ksi.ksi_addr = (void *)td->td_frame->tf_rip; trapsignal(td, &ksi); } Modified: stable/8/sys/amd64/ia32/ia32_syscall.c ============================================================================== --- stable/8/sys/amd64/ia32/ia32_syscall.c Wed Sep 28 21:42:13 2011 (r225854) +++ stable/8/sys/amd64/ia32/ia32_syscall.c Thu Sep 29 00:39:56 2011 (r225855) @@ -156,6 +156,8 @@ ia32_fetch_syscall_args(struct thread *t return (error); } +#include "../../kern/subr_syscall.c" + void ia32_syscall(struct trapframe *frame) { Modified: stable/8/sys/amd64/include/md_var.h ============================================================================== --- stable/8/sys/amd64/include/md_var.h Wed Sep 28 21:42:13 2011 (r225854) +++ stable/8/sys/amd64/include/md_var.h Thu Sep 29 00:39:56 2011 (r225855) @@ -75,6 +75,7 @@ struct fpreg; struct dbreg; struct dumperinfo; +void amd64_syscall(struct thread *td, int traced); void busdma_swi(void); void cpu_setregs(void); void doreti_iret(void) __asm(__STRING(doreti_iret)); Modified: stable/8/sys/i386/i386/trap.c ============================================================================== --- stable/8/sys/i386/i386/trap.c Wed Sep 28 21:42:13 2011 (r225854) +++ stable/8/sys/i386/i386/trap.c Thu Sep 29 00:39:56 2011 (r225855) @@ -1060,6 +1060,8 @@ cpu_fetch_syscall_args(struct thread *td return (error); } +#include "../../kern/subr_syscall.c" + /* * syscall - system call request C handler * Modified: stable/8/sys/ia64/ia32/ia32_trap.c ============================================================================== --- stable/8/sys/ia64/ia32/ia32_trap.c Wed Sep 28 21:42:13 2011 (r225854) +++ stable/8/sys/ia64/ia32/ia32_trap.c Thu Sep 29 00:39:56 2011 (r225855) @@ -144,6 +144,8 @@ ia32_fetch_syscall_args(struct thread *t return (error); } +#include "../../kern/subr_syscall.c" + static void ia32_syscall(struct trapframe *tf) { Modified: stable/8/sys/ia64/ia64/trap.c ============================================================================== --- stable/8/sys/ia64/ia64/trap.c Wed Sep 28 21:42:13 2011 (r225854) +++ stable/8/sys/ia64/ia64/trap.c Thu Sep 29 00:39:56 2011 (r225855) @@ -936,6 +936,8 @@ cpu_fetch_syscall_args(struct thread *td return (0); } +#include "../../kern/subr_syscall.c" + /* * Process a system call. * Copied and modified: stable/8/sys/kern/subr_syscall.c (from r225474, head/sys/kern/subr_syscall.c) ============================================================================== --- head/sys/kern/subr_syscall.c Sun Sep 11 16:05:09 2011 (r225474, copy source) +++ stable/8/sys/kern/subr_syscall.c Thu Sep 29 00:39:56 2011 (r225855) @@ -38,13 +38,11 @@ * from: @(#)trap.c 7.4 (Berkeley) 5/13/91 */ -#include "opt_capsicum.h" #include "opt_ktrace.h" #include "opt_kdtrace.h" __FBSDID("$FreeBSD$"); -#include <sys/capability.h> #include <sys/ktr.h> #ifdef KTRACE #include <sys/uio.h> @@ -100,22 +98,6 @@ syscallenter(struct thread *td, struct s goto retval; } -#ifdef CAPABILITY_MODE - /* - * In capability mode, we only allow access to system calls - * flagged with SYF_CAPENABLED. - */ - if (IN_CAPABILITY_MODE(td) && - !(sa->callp->sy_flags & SYF_CAPENABLED)) { - error = ECAPMODE; - goto retval; - } -#endif - - error = syscall_thread_enter(td, sa->callp); - if (error != 0) - goto retval; - #ifdef KDTRACE_HOOKS /* * If the systrace module has registered it's probe @@ -144,7 +126,6 @@ syscallenter(struct thread *td, struct s (*systrace_probe_func)(sa->callp->sy_return, sa->code, sa->callp, NULL, (error) ? -1 : td->td_retval[0]); #endif - syscall_thread_exit(td, sa->callp); CTR4(KTR_SYSC, "syscall: p=%p error=%d return %#lx %#lx", p, error, td->td_retval[0], td->td_retval[1]); } Modified: stable/8/sys/kern/subr_trap.c ============================================================================== --- stable/8/sys/kern/subr_trap.c Wed Sep 28 21:42:13 2011 (r225854) +++ stable/8/sys/kern/subr_trap.c Thu Sep 29 00:39:56 2011 (r225855) @@ -249,7 +249,6 @@ ast(struct trapframe *framep) mtx_assert(&Giant, MA_NOTOWNED); } -#ifdef HAVE_SYSCALL_ARGS_DEF const char * syscallname(struct proc *p, u_int code) { @@ -261,148 +260,3 @@ syscallname(struct proc *p, u_int code) return (unknown); return (sv->sv_syscallnames[code]); } - -int -syscallenter(struct thread *td, struct syscall_args *sa) -{ - struct proc *p; - int error, traced; - - PCPU_INC(cnt.v_syscall); - p = td->td_proc; - td->td_syscalls++; - - td->td_pticks = 0; - if (td->td_ucred != p->p_ucred) - cred_update_thread(td); - if (p->p_flag & P_TRACED) { - traced = 1; - PROC_LOCK(p); - td->td_dbgflags &= ~TDB_USERWR; - td->td_dbgflags |= TDB_SCE; - PROC_UNLOCK(p); - } else - traced = 0; - error = (p->p_sysent->sv_fetch_syscall_args)(td, sa); -#ifdef KTRACE - if (KTRPOINT(td, KTR_SYSCALL)) - ktrsyscall(sa->code, sa->narg, sa->args); -#endif - - CTR6(KTR_SYSC, -"syscall: td=%p pid %d %s (%#lx, %#lx, %#lx)", - td, td->td_proc->p_pid, syscallname(p, sa->code), - sa->args[0], sa->args[1], sa->args[2]); - - if (error == 0) { - STOPEVENT(p, S_SCE, sa->narg); - PTRACESTOP_SC(p, td, S_PT_SCE); - if (td->td_dbgflags & TDB_USERWR) { - /* - * Reread syscall number and arguments if - * debugger modified registers or memory. - */ - error = (p->p_sysent->sv_fetch_syscall_args)(td, sa); -#ifdef KTRACE - if (KTRPOINT(td, KTR_SYSCALL)) - ktrsyscall(sa->code, sa->narg, sa->args); -#endif - if (error != 0) - goto retval; - } - -#ifdef KDTRACE_HOOKS - /* - * If the systrace module has registered it's probe - * callback and if there is a probe active for the - * syscall 'entry', process the probe. - */ - if (systrace_probe_func != NULL && sa->callp->sy_entry != 0) - (*systrace_probe_func)(sa->callp->sy_entry, sa->code, - sa->callp, sa->args, 0); -#endif - - AUDIT_SYSCALL_ENTER(sa->code, td); - error = (sa->callp->sy_call)(td, sa->args); - AUDIT_SYSCALL_EXIT(error, td); - - /* Save the latest error return value. */ - td->td_errno = error; - -#ifdef KDTRACE_HOOKS - /* - * If the systrace module has registered it's probe - * callback and if there is a probe active for the - * syscall 'return', process the probe. - */ - if (systrace_probe_func != NULL && sa->callp->sy_return != 0) - (*systrace_probe_func)(sa->callp->sy_return, sa->code, - sa->callp, NULL, (error) ? -1 : td->td_retval[0]); -#endif - CTR4(KTR_SYSC, "syscall: p=%p error=%d return %#lx %#lx", - p, error, td->td_retval[0], td->td_retval[1]); - } - retval: - if (traced) { - PROC_LOCK(p); - td->td_dbgflags &= ~TDB_SCE; - PROC_UNLOCK(p); - } - (p->p_sysent->sv_set_syscall_retval)(td, error); - return (error); -} - -void -syscallret(struct thread *td, int error, struct syscall_args *sa __unused) -{ - struct proc *p; - int traced; - - p = td->td_proc; - - /* - * Check for misbehavior. - */ - WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning", - syscallname(p, sa->code)); - KASSERT(td->td_critnest == 0, - ("System call %s returning in a critical section", - syscallname(p, sa->code))); - KASSERT(td->td_locks == 0, - ("System call %s returning with %d locks held", - syscallname(p, sa->code), td->td_locks)); - - /* - * Handle reschedule and other end-of-syscall issues - */ - userret(td, td->td_frame); - - CTR4(KTR_SYSC, "syscall %s exit thread %p pid %d proc %s", - syscallname(p, sa->code), td, td->td_proc->p_pid, td->td_name); - -#ifdef KTRACE - if (KTRPOINT(td, KTR_SYSRET)) - ktrsysret(sa->code, error, td->td_retval[0]); -#endif - - if (p->p_flag & P_TRACED) { - traced = 1; - PROC_LOCK(p); - td->td_dbgflags |= TDB_SCX; - PROC_UNLOCK(p); - } else - traced = 0; - /* - * This works because errno is findable through the - * register set. If we ever support an emulation where this - * is not the case, this code will need to be revisited. - */ - STOPEVENT(p, S_SCX, sa->code); - PTRACESTOP_SC(p, td, S_PT_SCX); - if (traced || (td->td_dbgflags & (TDB_EXEC | TDB_FORK)) != 0) { - PROC_LOCK(p); - td->td_dbgflags &= ~(TDB_SCX | TDB_EXEC | TDB_FORK); - PROC_UNLOCK(p); - } -} -#endif /* HAVE_SYSCALL_ARGS_DEF */ Modified: stable/8/sys/powerpc/aim/trap.c ============================================================================== --- stable/8/sys/powerpc/aim/trap.c Wed Sep 28 21:42:13 2011 (r225854) +++ stable/8/sys/powerpc/aim/trap.c Thu Sep 29 00:39:56 2011 (r225855) @@ -394,6 +394,8 @@ cpu_fetch_syscall_args(struct thread *td return (error); } +#include "../../kern/subr_syscall.c" + void syscall(struct trapframe *frame) { Modified: stable/8/sys/powerpc/booke/trap.c ============================================================================== --- stable/8/sys/powerpc/booke/trap.c Wed Sep 28 21:42:13 2011 (r225854) +++ stable/8/sys/powerpc/booke/trap.c Thu Sep 29 00:39:56 2011 (r225855) @@ -379,6 +379,8 @@ cpu_fetch_syscall_args(struct thread *td return (error); } +#include "../../kern/subr_syscall.c" + void syscall(struct trapframe *frame) { Modified: stable/8/sys/sparc64/sparc64/trap.c ============================================================================== --- stable/8/sys/sparc64/sparc64/trap.c Wed Sep 28 21:42:13 2011 (r225854) +++ stable/8/sys/sparc64/sparc64/trap.c Thu Sep 29 00:39:56 2011 (r225855) @@ -614,6 +614,8 @@ cpu_fetch_syscall_args(struct thread *td return (error); } +#include "../../kern/subr_syscall.c" + /* * Syscall handler * The arguments to the syscall are passed in the out registers by the caller, Modified: stable/8/sys/sun4v/sun4v/trap.c ============================================================================== --- stable/8/sys/sun4v/sun4v/trap.c Wed Sep 28 21:42:13 2011 (r225854) +++ stable/8/sys/sun4v/sun4v/trap.c Thu Sep 29 00:39:56 2011 (r225855) @@ -610,6 +610,8 @@ cpu_fetch_syscall_args(struct thread *td return (error); } +#include "../../kern/subr_syscall.c" + /* * Syscall handler. The arguments to the syscall are passed in the o registers * by the caller, and are saved in the trap frame. The syscall number is passed Modified: stable/8/sys/sys/proc.h ============================================================================== --- stable/8/sys/sys/proc.h Wed Sep 28 21:42:13 2011 (r225854) +++ stable/8/sys/sys/proc.h Thu Sep 29 00:39:56 2011 (r225855) @@ -853,9 +853,6 @@ void cpu_switch(struct thread *, struct void cpu_throw(struct thread *, struct thread *) __dead2; void unsleep(struct thread *); void userret(struct thread *, struct trapframe *); -struct syscall_args; -int syscallenter(struct thread *, struct syscall_args *); -void syscallret(struct thread *, int, struct syscall_args *); void cpu_exit(struct thread *); void exit1(struct thread *, int) __dead2;
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201109290039.p8T0dvEo096785>