Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 29 Sep 2011 00:39:57 +0000 (UTC)
From:      Konstantin Belousov <kib@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-8@freebsd.org
Subject:   svn commit: r225855 - in stable/8/sys: amd64/amd64 amd64/ia32 amd64/include i386/i386 ia64/ia32 ia64/ia64 kern powerpc/aim powerpc/booke sparc64/sparc64 sun4v/sun4v sys
Message-ID:  <201109290039.p8T0dvEo096785@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: kib
Date: Thu Sep 29 00:39:56 2011
New Revision: 225855
URL: http://svn.freebsd.org/changeset/base/225855

Log:
  Merge the optimizations for the syscall entry and leave.
  
  MFC r225474:
  Inline the syscallenter() and syscallret(). This reduces the time measured
  by the syscall entry speed microbenchmarks by ~10% on amd64.
  
  MFC r225475:
  Perform amd64-specific microoptimizations for native syscall entry
  sequence. The effect is ~1% on the microbenchmark.
  
  In particular, do not restore registers which are preserved by the
  C calling sequence. Align the jump target. Avoid unneeded memory
  accesses by calculating some data in syscall entry trampoline.
  
  MFC r225483:
  The jump target shall be after the padding, not into it.
  
  MFC r225575:
  Microoptimize the return path for the fast syscalls on amd64. Arrange
  the code to have the fall-through path to follow the likely target.
  Do not use intermediate register to reload user %rsp.
  
  MFC r225576:
  Put amd64_syscall() prototype in md_var.h.
  
  Tested by:	 Alexandr Kovalenko <never nevermind kiev ua>

Added:
  stable/8/sys/kern/subr_syscall.c
     - copied, changed from r225474, head/sys/kern/subr_syscall.c
Modified:
  stable/8/sys/amd64/amd64/exception.S
  stable/8/sys/amd64/amd64/genassym.c
  stable/8/sys/amd64/amd64/trap.c
  stable/8/sys/amd64/ia32/ia32_syscall.c
  stable/8/sys/amd64/include/md_var.h
  stable/8/sys/i386/i386/trap.c
  stable/8/sys/ia64/ia32/ia32_trap.c
  stable/8/sys/ia64/ia64/trap.c
  stable/8/sys/kern/subr_trap.c
  stable/8/sys/powerpc/aim/trap.c
  stable/8/sys/powerpc/booke/trap.c
  stable/8/sys/sparc64/sparc64/trap.c
  stable/8/sys/sun4v/sun4v/trap.c
  stable/8/sys/sys/proc.h
Directory Properties:
  stable/8/sys/   (props changed)
  stable/8/sys/amd64/include/xen/   (props changed)
  stable/8/sys/cddl/contrib/opensolaris/   (props changed)
  stable/8/sys/contrib/dev/acpica/   (props changed)
  stable/8/sys/contrib/pf/   (props changed)

Modified: stable/8/sys/amd64/amd64/exception.S
==============================================================================
--- stable/8/sys/amd64/amd64/exception.S	Wed Sep 28 21:42:13 2011	(r225854)
+++ stable/8/sys/amd64/amd64/exception.S	Thu Sep 29 00:39:56 2011	(r225855)
@@ -380,8 +380,11 @@ IDTVEC(fast_syscall)
 	movl	$TF_HASSEGS,TF_FLAGS(%rsp)
 	cld
 	FAKE_MCOUNT(TF_RIP(%rsp))
-	movq	%rsp,%rdi
-	call	syscall
+	movq	PCPU(CURTHREAD),%rdi
+	movq	%rsp,TD_FRAME(%rdi)
+	movl	TF_RFLAGS(%rsp),%esi
+	andl	$PSL_T,%esi
+	call	amd64_syscall
 1:	movq	PCPU(CURPCB),%rax
 	/* Disable interrupts before testing PCB_FULL_IRET. */
 	cli
@@ -390,29 +393,25 @@ IDTVEC(fast_syscall)
 	/* Check for and handle AST's on return to userland. */
 	movq	PCPU(CURTHREAD),%rax
 	testl	$TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax)
-	je	2f
-	sti
-	movq	%rsp, %rdi
-	call	ast
-	jmp	1b
-2:	/* Restore preserved registers. */
+	jne	2f
+	/* Restore preserved registers. */
 	MEXITCOUNT
 	movq	TF_RDI(%rsp),%rdi	/* bonus; preserve arg 1 */
 	movq	TF_RSI(%rsp),%rsi	/* bonus: preserve arg 2 */
 	movq	TF_RDX(%rsp),%rdx	/* return value 2 */
 	movq	TF_RAX(%rsp),%rax	/* return value 1 */
-	movq	TF_RBX(%rsp),%rbx	/* C preserved */
-	movq	TF_RBP(%rsp),%rbp	/* C preserved */
-	movq	TF_R12(%rsp),%r12	/* C preserved */
-	movq	TF_R13(%rsp),%r13	/* C preserved */
-	movq	TF_R14(%rsp),%r14	/* C preserved */
-	movq	TF_R15(%rsp),%r15	/* C preserved */
 	movq	TF_RFLAGS(%rsp),%r11	/* original %rflags */
 	movq	TF_RIP(%rsp),%rcx	/* original %rip */
-	movq	TF_RSP(%rsp),%r9	/* user stack pointer */
-	movq	%r9,%rsp		/* original %rsp */
+	movq	TF_RSP(%rsp),%rsp	/* user stack pointer */
 	swapgs
 	sysretq
+
+2:	/* AST scheduled. */
+	sti
+	movq	%rsp,%rdi
+	call	ast
+	jmp	1b
+
 3:	/* Requested full context restore, use doreti for that. */
 	MEXITCOUNT
 	jmp	doreti

Modified: stable/8/sys/amd64/amd64/genassym.c
==============================================================================
--- stable/8/sys/amd64/amd64/genassym.c	Wed Sep 28 21:42:13 2011	(r225854)
+++ stable/8/sys/amd64/amd64/genassym.c	Thu Sep 29 00:39:56 2011	(r225855)
@@ -87,6 +87,7 @@ ASSYM(TD_PCB, offsetof(struct thread, td
 ASSYM(TD_PFLAGS, offsetof(struct thread, td_pflags));
 ASSYM(TD_PROC, offsetof(struct thread, td_proc));
 ASSYM(TD_TID, offsetof(struct thread, td_tid));
+ASSYM(TD_FRAME, offsetof(struct thread, td_frame));
 
 ASSYM(TDF_ASTPENDING, TDF_ASTPENDING);
 ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED);

Modified: stable/8/sys/amd64/amd64/trap.c
==============================================================================
--- stable/8/sys/amd64/amd64/trap.c	Wed Sep 28 21:42:13 2011	(r225854)
+++ stable/8/sys/amd64/amd64/trap.c	Thu Sep 29 00:39:56 2011	(r225855)
@@ -889,41 +889,37 @@ cpu_fetch_syscall_args(struct thread *td
 	return (error);
 }
 
+#include "../../kern/subr_syscall.c"
+
 /*
  *	syscall -	system call request C handler
  *
  *	A system call is essentially treated as a trap.
  */
 void
-syscall(struct trapframe *frame)
+amd64_syscall(struct thread *td, int traced)
 {
-	struct thread *td;
 	struct syscall_args sa;
-	register_t orig_tf_rflags;
 	int error;
 	ksiginfo_t ksi;
 
 #ifdef DIAGNOSTIC
-	if (ISPL(frame->tf_cs) != SEL_UPL) {
+	if (ISPL(td->td_frame->tf_cs) != SEL_UPL) {
 		panic("syscall");
 		/* NOT REACHED */
 	}
 #endif
-	orig_tf_rflags = frame->tf_rflags;
-	td = curthread;
-	td->td_frame = frame;
-
 	error = syscallenter(td, &sa);
 
 	/*
 	 * Traced syscall.
 	 */
-	if (orig_tf_rflags & PSL_T) {
-		frame->tf_rflags &= ~PSL_T;
+	if (__predict_false(traced)) {
+		td->td_frame->tf_rflags &= ~PSL_T;
 		ksiginfo_init_trap(&ksi);
 		ksi.ksi_signo = SIGTRAP;
 		ksi.ksi_code = TRAP_TRACE;
-		ksi.ksi_addr = (void *)frame->tf_rip;
+		ksi.ksi_addr = (void *)td->td_frame->tf_rip;
 		trapsignal(td, &ksi);
 	}
 

Modified: stable/8/sys/amd64/ia32/ia32_syscall.c
==============================================================================
--- stable/8/sys/amd64/ia32/ia32_syscall.c	Wed Sep 28 21:42:13 2011	(r225854)
+++ stable/8/sys/amd64/ia32/ia32_syscall.c	Thu Sep 29 00:39:56 2011	(r225855)
@@ -156,6 +156,8 @@ ia32_fetch_syscall_args(struct thread *t
 	return (error);
 }
 
+#include "../../kern/subr_syscall.c"
+
 void
 ia32_syscall(struct trapframe *frame)
 {

Modified: stable/8/sys/amd64/include/md_var.h
==============================================================================
--- stable/8/sys/amd64/include/md_var.h	Wed Sep 28 21:42:13 2011	(r225854)
+++ stable/8/sys/amd64/include/md_var.h	Thu Sep 29 00:39:56 2011	(r225855)
@@ -75,6 +75,7 @@ struct	fpreg;
 struct  dbreg;
 struct	dumperinfo;
 
+void	amd64_syscall(struct thread *td, int traced);
 void	busdma_swi(void);
 void	cpu_setregs(void);
 void	doreti_iret(void) __asm(__STRING(doreti_iret));

Modified: stable/8/sys/i386/i386/trap.c
==============================================================================
--- stable/8/sys/i386/i386/trap.c	Wed Sep 28 21:42:13 2011	(r225854)
+++ stable/8/sys/i386/i386/trap.c	Thu Sep 29 00:39:56 2011	(r225855)
@@ -1060,6 +1060,8 @@ cpu_fetch_syscall_args(struct thread *td
 	return (error);
 }
 
+#include "../../kern/subr_syscall.c"
+
 /*
  *	syscall -	system call request C handler
  *

Modified: stable/8/sys/ia64/ia32/ia32_trap.c
==============================================================================
--- stable/8/sys/ia64/ia32/ia32_trap.c	Wed Sep 28 21:42:13 2011	(r225854)
+++ stable/8/sys/ia64/ia32/ia32_trap.c	Thu Sep 29 00:39:56 2011	(r225855)
@@ -144,6 +144,8 @@ ia32_fetch_syscall_args(struct thread *t
 	return (error);
 }
 
+#include "../../kern/subr_syscall.c"
+
 static void
 ia32_syscall(struct trapframe *tf)
 {

Modified: stable/8/sys/ia64/ia64/trap.c
==============================================================================
--- stable/8/sys/ia64/ia64/trap.c	Wed Sep 28 21:42:13 2011	(r225854)
+++ stable/8/sys/ia64/ia64/trap.c	Thu Sep 29 00:39:56 2011	(r225855)
@@ -936,6 +936,8 @@ cpu_fetch_syscall_args(struct thread *td
 	return (0);
 }
 
+#include "../../kern/subr_syscall.c"
+
 /*
  * Process a system call.
  *

Copied and modified: stable/8/sys/kern/subr_syscall.c (from r225474, head/sys/kern/subr_syscall.c)
==============================================================================
--- head/sys/kern/subr_syscall.c	Sun Sep 11 16:05:09 2011	(r225474, copy source)
+++ stable/8/sys/kern/subr_syscall.c	Thu Sep 29 00:39:56 2011	(r225855)
@@ -38,13 +38,11 @@
  *	from: @(#)trap.c	7.4 (Berkeley) 5/13/91
  */
 
-#include "opt_capsicum.h"
 #include "opt_ktrace.h"
 #include "opt_kdtrace.h"
 
 __FBSDID("$FreeBSD$");
 
-#include <sys/capability.h>
 #include <sys/ktr.h>
 #ifdef KTRACE
 #include <sys/uio.h>
@@ -100,22 +98,6 @@ syscallenter(struct thread *td, struct s
 				goto retval;
 		}
 
-#ifdef CAPABILITY_MODE
-		/*
-		 * In capability mode, we only allow access to system calls
-		 * flagged with SYF_CAPENABLED.
-		 */
-		if (IN_CAPABILITY_MODE(td) &&
-		    !(sa->callp->sy_flags & SYF_CAPENABLED)) {
-			error = ECAPMODE;
-			goto retval;
-		}
-#endif
-
-		error = syscall_thread_enter(td, sa->callp);
-		if (error != 0)
-			goto retval;
-
 #ifdef KDTRACE_HOOKS
 		/*
 		 * If the systrace module has registered it's probe
@@ -144,7 +126,6 @@ syscallenter(struct thread *td, struct s
 			(*systrace_probe_func)(sa->callp->sy_return, sa->code,
 			    sa->callp, NULL, (error) ? -1 : td->td_retval[0]);
 #endif
-		syscall_thread_exit(td, sa->callp);
 		CTR4(KTR_SYSC, "syscall: p=%p error=%d return %#lx %#lx",
 		    p, error, td->td_retval[0], td->td_retval[1]);
 	}

Modified: stable/8/sys/kern/subr_trap.c
==============================================================================
--- stable/8/sys/kern/subr_trap.c	Wed Sep 28 21:42:13 2011	(r225854)
+++ stable/8/sys/kern/subr_trap.c	Thu Sep 29 00:39:56 2011	(r225855)
@@ -249,7 +249,6 @@ ast(struct trapframe *framep)
 	mtx_assert(&Giant, MA_NOTOWNED);
 }
 
-#ifdef HAVE_SYSCALL_ARGS_DEF
 const char *
 syscallname(struct proc *p, u_int code)
 {
@@ -261,148 +260,3 @@ syscallname(struct proc *p, u_int code)
 		return (unknown);
 	return (sv->sv_syscallnames[code]);
 }
-
-int
-syscallenter(struct thread *td, struct syscall_args *sa)
-{
-	struct proc *p;
-	int error, traced;
-
-	PCPU_INC(cnt.v_syscall);
-	p = td->td_proc;
-	td->td_syscalls++;
-
-	td->td_pticks = 0;
-	if (td->td_ucred != p->p_ucred)
-		cred_update_thread(td);
-	if (p->p_flag & P_TRACED) {
-		traced = 1;
-		PROC_LOCK(p);
-		td->td_dbgflags &= ~TDB_USERWR;
-		td->td_dbgflags |= TDB_SCE;
-		PROC_UNLOCK(p);
-	} else
-		traced = 0;
-	error = (p->p_sysent->sv_fetch_syscall_args)(td, sa);
-#ifdef KTRACE
-	if (KTRPOINT(td, KTR_SYSCALL))
-		ktrsyscall(sa->code, sa->narg, sa->args);
-#endif
-
-	CTR6(KTR_SYSC,
-"syscall: td=%p pid %d %s (%#lx, %#lx, %#lx)",
-	    td, td->td_proc->p_pid, syscallname(p, sa->code),
-	    sa->args[0], sa->args[1], sa->args[2]);
-
-	if (error == 0) {
-		STOPEVENT(p, S_SCE, sa->narg);
-		PTRACESTOP_SC(p, td, S_PT_SCE);
-		if (td->td_dbgflags & TDB_USERWR) {
-			/*
-			 * Reread syscall number and arguments if
-			 * debugger modified registers or memory.
-			 */
-			error = (p->p_sysent->sv_fetch_syscall_args)(td, sa);
-#ifdef KTRACE
-			if (KTRPOINT(td, KTR_SYSCALL))
-				ktrsyscall(sa->code, sa->narg, sa->args);
-#endif
-			if (error != 0)
-				goto retval;
-		}
-
-#ifdef KDTRACE_HOOKS
-		/*
-		 * If the systrace module has registered it's probe
-		 * callback and if there is a probe active for the
-		 * syscall 'entry', process the probe.
-		 */
-		if (systrace_probe_func != NULL && sa->callp->sy_entry != 0)
-			(*systrace_probe_func)(sa->callp->sy_entry, sa->code,
-			    sa->callp, sa->args, 0);
-#endif
-
-		AUDIT_SYSCALL_ENTER(sa->code, td);
-		error = (sa->callp->sy_call)(td, sa->args);
-		AUDIT_SYSCALL_EXIT(error, td);
-
-		/* Save the latest error return value. */
-		td->td_errno = error;
-
-#ifdef KDTRACE_HOOKS
-		/*
-		 * If the systrace module has registered it's probe
-		 * callback and if there is a probe active for the
-		 * syscall 'return', process the probe.
-		 */
-		if (systrace_probe_func != NULL && sa->callp->sy_return != 0)
-			(*systrace_probe_func)(sa->callp->sy_return, sa->code,
-			    sa->callp, NULL, (error) ? -1 : td->td_retval[0]);
-#endif
-		CTR4(KTR_SYSC, "syscall: p=%p error=%d return %#lx %#lx",
-		    p, error, td->td_retval[0], td->td_retval[1]);
-	}
- retval:
-	if (traced) {
-		PROC_LOCK(p);
-		td->td_dbgflags &= ~TDB_SCE;
-		PROC_UNLOCK(p);
-	}
-	(p->p_sysent->sv_set_syscall_retval)(td, error);
-	return (error);
-}
-
-void
-syscallret(struct thread *td, int error, struct syscall_args *sa __unused)
-{
-	struct proc *p;
-	int traced;
-
-	p = td->td_proc;
-
-	/*
-	 * Check for misbehavior.
-	 */
-	WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning",
-	    syscallname(p, sa->code));
-	KASSERT(td->td_critnest == 0,
-	    ("System call %s returning in a critical section",
-	    syscallname(p, sa->code)));
-	KASSERT(td->td_locks == 0,
-	    ("System call %s returning with %d locks held",
-	     syscallname(p, sa->code), td->td_locks));
-
-	/*
-	 * Handle reschedule and other end-of-syscall issues
-	 */
-	userret(td, td->td_frame);
-
-	CTR4(KTR_SYSC, "syscall %s exit thread %p pid %d proc %s",
-	    syscallname(p, sa->code), td, td->td_proc->p_pid, td->td_name);
-
-#ifdef KTRACE
-	if (KTRPOINT(td, KTR_SYSRET))
-		ktrsysret(sa->code, error, td->td_retval[0]);
-#endif
-
-	if (p->p_flag & P_TRACED) {
-		traced = 1;
-		PROC_LOCK(p);
-		td->td_dbgflags |= TDB_SCX;
-		PROC_UNLOCK(p);
-	} else
-		traced = 0;
-	/*
-	 * This works because errno is findable through the
-	 * register set.  If we ever support an emulation where this
-	 * is not the case, this code will need to be revisited.
-	 */
-	STOPEVENT(p, S_SCX, sa->code);
-	PTRACESTOP_SC(p, td, S_PT_SCX);
-	if (traced || (td->td_dbgflags & (TDB_EXEC | TDB_FORK)) != 0) {
-		PROC_LOCK(p);
-		td->td_dbgflags &= ~(TDB_SCX | TDB_EXEC | TDB_FORK);
-		PROC_UNLOCK(p);
-	}
-}
-#endif /* HAVE_SYSCALL_ARGS_DEF */

Modified: stable/8/sys/powerpc/aim/trap.c
==============================================================================
--- stable/8/sys/powerpc/aim/trap.c	Wed Sep 28 21:42:13 2011	(r225854)
+++ stable/8/sys/powerpc/aim/trap.c	Thu Sep 29 00:39:56 2011	(r225855)
@@ -394,6 +394,8 @@ cpu_fetch_syscall_args(struct thread *td
 	return (error);
 }
 
+#include "../../kern/subr_syscall.c"
+
 void
 syscall(struct trapframe *frame)
 {

Modified: stable/8/sys/powerpc/booke/trap.c
==============================================================================
--- stable/8/sys/powerpc/booke/trap.c	Wed Sep 28 21:42:13 2011	(r225854)
+++ stable/8/sys/powerpc/booke/trap.c	Thu Sep 29 00:39:56 2011	(r225855)
@@ -379,6 +379,8 @@ cpu_fetch_syscall_args(struct thread *td
 	return (error);
 }
 
+#include "../../kern/subr_syscall.c"
+
 void
 syscall(struct trapframe *frame)
 {

Modified: stable/8/sys/sparc64/sparc64/trap.c
==============================================================================
--- stable/8/sys/sparc64/sparc64/trap.c	Wed Sep 28 21:42:13 2011	(r225854)
+++ stable/8/sys/sparc64/sparc64/trap.c	Thu Sep 29 00:39:56 2011	(r225855)
@@ -614,6 +614,8 @@ cpu_fetch_syscall_args(struct thread *td
 	return (error);
 }
 
+#include "../../kern/subr_syscall.c"
+
 /*
  * Syscall handler
  * The arguments to the syscall are passed in the out registers by the caller,

Modified: stable/8/sys/sun4v/sun4v/trap.c
==============================================================================
--- stable/8/sys/sun4v/sun4v/trap.c	Wed Sep 28 21:42:13 2011	(r225854)
+++ stable/8/sys/sun4v/sun4v/trap.c	Thu Sep 29 00:39:56 2011	(r225855)
@@ -610,6 +610,8 @@ cpu_fetch_syscall_args(struct thread *td
 	return (error);
 }
 
+#include "../../kern/subr_syscall.c"
+
 /*
  * Syscall handler. The arguments to the syscall are passed in the o registers
  * by the caller, and are saved in the trap frame. The syscall number is passed

Modified: stable/8/sys/sys/proc.h
==============================================================================
--- stable/8/sys/sys/proc.h	Wed Sep 28 21:42:13 2011	(r225854)
+++ stable/8/sys/sys/proc.h	Thu Sep 29 00:39:56 2011	(r225855)
@@ -853,9 +853,6 @@ void	cpu_switch(struct thread *, struct 
 void	cpu_throw(struct thread *, struct thread *) __dead2;
 void	unsleep(struct thread *);
 void	userret(struct thread *, struct trapframe *);
-struct syscall_args;
-int	syscallenter(struct thread *, struct syscall_args *);
-void	syscallret(struct thread *, int, struct syscall_args *);
 
 void	cpu_exit(struct thread *);
 void	exit1(struct thread *, int) __dead2;



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201109290039.p8T0dvEo096785>