From owner-svn-src-all@FreeBSD.ORG Thu Apr 7 21:32:25 2011 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 6EC35106564A; Thu, 7 Apr 2011 21:32:25 +0000 (UTC) (envelope-from jhb@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 44E698FC14; Thu, 7 Apr 2011 21:32:25 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id p37LWP6w052538; Thu, 7 Apr 2011 21:32:25 GMT (envelope-from jhb@svn.freebsd.org) Received: (from jhb@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id p37LWPuu052536; Thu, 7 Apr 2011 21:32:25 GMT (envelope-from jhb@svn.freebsd.org) Message-Id: <201104072132.p37LWPuu052536@svn.freebsd.org> From: John Baldwin Date: Thu, 7 Apr 2011 21:32:25 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r220430 - head/sys/amd64/amd64 X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 07 Apr 2011 21:32:25 -0000 Author: jhb Date: Thu Apr 7 21:32:25 2011 New Revision: 220430 URL: http://svn.freebsd.org/changeset/base/220430 Log: If a system call does not request a full interrupt return, use a fast path via the sysretq instruction to return from the system call. This was removed in 190620 and not quite fully restored in 195486. This resolves most of the performance regression in system call microbenchmarks between 7 and 8 on amd64. Reviewed by: kib MFC after: 1 week Modified: head/sys/amd64/amd64/exception.S Modified: head/sys/amd64/amd64/exception.S ============================================================================== --- head/sys/amd64/amd64/exception.S Thu Apr 7 21:29:34 2011 (r220429) +++ head/sys/amd64/amd64/exception.S Thu Apr 7 21:32:25 2011 (r220430) @@ -339,6 +339,9 @@ IDTVEC(prot) * and the new privilige level. We are still running on the old user stack * pointer. We have to juggle a few things around to find our stack etc. * swapgs gives us access to our PCPU space only. + * + * We do not support invoking this from a custom %cs or %ss (e.g. using + * entries from an LDT). */ IDTVEC(fast_syscall) swapgs @@ -380,6 +383,36 @@ IDTVEC(fast_syscall) movq %rsp,%rdi call syscall movq PCPU(CURPCB),%rax + testq $PCB_FULL_IRET,PCB_FLAGS(%rax) + jne 3f +1: /* Check for and handle AST's on return to userland. */ + cli + movq PCPU(CURTHREAD),%rax + testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) + je 2f + sti + movq %rsp, %rdi + call ast + jmp 1b +2: /* Restore preserved registers. */ + MEXITCOUNT + movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ + movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ + movq TF_RDX(%rsp),%rdx /* return value 2 */ + movq TF_RAX(%rsp),%rax /* return value 1 */ + movq TF_RBX(%rsp),%rbx /* C preserved */ + movq TF_RBP(%rsp),%rbp /* C preserved */ + movq TF_R12(%rsp),%r12 /* C preserved */ + movq TF_R13(%rsp),%r13 /* C preserved */ + movq TF_R14(%rsp),%r14 /* C preserved */ + movq TF_R15(%rsp),%r15 /* C preserved */ + movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ + movq TF_RIP(%rsp),%rcx /* original %rip */ + movq TF_RSP(%rsp),%r9 /* user stack pointer */ + movq %r9,%rsp /* original %rsp */ + swapgs + sysretq +3: /* Requested full context restore, use doreti for that. */ MEXITCOUNT jmp doreti