From owner-svn-src-stable-8@FreeBSD.ORG Mon Apr 18 18:55:28 2011 Return-Path: Delivered-To: svn-src-stable-8@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 053B61065673; Mon, 18 Apr 2011 18:55:28 +0000 (UTC) (envelope-from jhb@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id DC7258FC14; Mon, 18 Apr 2011 18:55:27 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id p3IItRwk081182; Mon, 18 Apr 2011 18:55:27 GMT (envelope-from jhb@svn.freebsd.org) Received: (from jhb@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id p3IItRoL081180; Mon, 18 Apr 2011 18:55:27 GMT (envelope-from jhb@svn.freebsd.org) Message-Id: <201104181855.p3IItRoL081180@svn.freebsd.org> From: John Baldwin Date: Mon, 18 Apr 2011 18:55:27 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-8@freebsd.org X-SVN-Group: stable-8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r220797 - stable/8/sys/amd64/amd64 X-BeenThere: svn-src-stable-8@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: SVN commit messages for only the 8-stable src tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 18 Apr 2011 18:55:28 -0000 Author: jhb Date: Mon Apr 18 18:55:27 2011 New Revision: 220797 URL: http://svn.freebsd.org/changeset/base/220797 Log: MFC 220430,220431,220452,220460: If a system call does not request a full interrupt return, use a fast path via the sysretq instruction to return from the system call. This resolves most of the performance regression in system call microbenchmarks between 7 and 8 on amd64. While here, trim an instruction (and memory access) from the doreti path and fix a typo in a comment. Modified: stable/8/sys/amd64/amd64/exception.S Directory Properties: stable/8/sys/ (props changed) stable/8/sys/amd64/include/xen/ (props changed) stable/8/sys/cddl/contrib/opensolaris/ (props changed) stable/8/sys/contrib/dev/acpica/ (props changed) stable/8/sys/contrib/pf/ (props changed) Modified: stable/8/sys/amd64/amd64/exception.S ============================================================================== --- stable/8/sys/amd64/amd64/exception.S Mon Apr 18 18:22:10 2011 (r220796) +++ stable/8/sys/amd64/amd64/exception.S Mon Apr 18 18:55:27 2011 (r220797) @@ -339,6 +339,9 @@ IDTVEC(prot) * and the new privilige level. We are still running on the old user stack * pointer. We have to juggle a few things around to find our stack etc. * swapgs gives us access to our PCPU space only. + * + * We do not support invoking this from a custom %cs or %ss (e.g. using + * entries from an LDT). */ IDTVEC(fast_syscall) swapgs @@ -379,7 +382,38 @@ IDTVEC(fast_syscall) FAKE_MCOUNT(TF_RIP(%rsp)) movq %rsp,%rdi call syscall - movq PCPU(CURPCB),%rax +1: movq PCPU(CURPCB),%rax + /* Disable interrupts before testing PCB_FULL_IRET. */ + cli + testl $PCB_FULL_IRET,PCB_FLAGS(%rax) + jnz 3f + /* Check for and handle AST's on return to userland. */ + movq PCPU(CURTHREAD),%rax + testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) + je 2f + sti + movq %rsp, %rdi + call ast + jmp 1b +2: /* Restore preserved registers. */ + MEXITCOUNT + movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ + movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ + movq TF_RDX(%rsp),%rdx /* return value 2 */ + movq TF_RAX(%rsp),%rax /* return value 1 */ + movq TF_RBX(%rsp),%rbx /* C preserved */ + movq TF_RBP(%rsp),%rbp /* C preserved */ + movq TF_R12(%rsp),%r12 /* C preserved */ + movq TF_R13(%rsp),%r13 /* C preserved */ + movq TF_R14(%rsp),%r14 /* C preserved */ + movq TF_R15(%rsp),%r15 /* C preserved */ + movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ + movq TF_RIP(%rsp),%rcx /* original %rip */ + movq TF_RSP(%rsp),%r9 /* user stack pointer */ + movq %r9,%rsp /* original %rsp */ + swapgs + sysretq +3: /* Requested full context restore, use doreti for that. */ MEXITCOUNT jmp doreti @@ -628,7 +662,7 @@ doreti: doreti_ast: /* * Check for ASTs atomically with returning. Disabling CPU - * interrupts provides sufficient locking eve in the SMP case, + * interrupts provides sufficient locking even in the SMP case, * since we will be informed of any new ASTs by an IPI. */ cli @@ -649,8 +683,7 @@ doreti_ast: */ doreti_exit: MEXITCOUNT - movq PCPU(CURTHREAD),%r8 - movq TD_PCB(%r8),%r8 + movq PCPU(CURPCB),%r8 /* * Do not reload segment registers for kernel.