From owner-svn-src-all@freebsd.org Sat Aug 3 17:07:05 2019 Return-Path: Delivered-To: svn-src-all@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id DE5E4B57FB; Sat, 3 Aug 2019 17:07:05 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) server-signature RSA-PSS (4096 bits) client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4619S95WkSz4Sqr; Sat, 3 Aug 2019 17:07:05 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 9B8911E21A; Sat, 3 Aug 2019 17:07:05 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id x73H75wI045857; Sat, 3 Aug 2019 17:07:05 GMT (envelope-from kib@FreeBSD.org) Received: (from kib@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id x73H75b3045855; Sat, 3 Aug 2019 17:07:05 GMT (envelope-from kib@FreeBSD.org) Message-Id: <201908031707.x73H75b3045855@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: kib set sender to kib@FreeBSD.org using -f From: Konstantin Belousov Date: Sat, 3 Aug 2019 17:07:05 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r350561 - in head/sys/amd64: amd64 include X-SVN-Group: head X-SVN-Commit-Author: kib X-SVN-Commit-Paths: in head/sys/amd64: amd64 include X-SVN-Commit-Revision: 350561 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 03 Aug 2019 17:07:05 -0000 Author: kib Date: Sat Aug 3 17:07:04 2019 New Revision: 350561 URL: https://svnweb.freebsd.org/changeset/base/350561 Log: amd64: Streamline exceptions and interrupts handlers. PTI-mode entry points were coded to set up the environment identical to non-PTI entry and then fall-through to non-PTI handlers, mostly. This has the drawback of requiring two more SWAPGS, first to access PCPU, and then to return to the state expected by the non-PTI entry point. Eliminate the duplication by doing more in entry stubs both for PTI and non-PTI, and adjusting the common code to expect that SWAPGS and some minimal registers saving is done by entries. Some less often used entries, in particular, #GP, #NP, and #SS, which can fault on doreti, are left as is because there are basically four variants of entrance, and they are not performance-critical, esp. comparing with e.g. #PF or interrupts. Reviewed by: markj (previous version) Tested by: pho (previous version) MFC after: 1 week Sponsored by: The FreeBSD Foundation Modified: head/sys/amd64/amd64/exception.S head/sys/amd64/include/asmacros.h Modified: head/sys/amd64/amd64/exception.S ============================================================================== --- head/sys/amd64/amd64/exception.S Sat Aug 3 16:57:14 2019 (r350560) +++ head/sys/amd64/amd64/exception.S Sat Aug 3 17:07:04 2019 (r350561) @@ -105,14 +105,31 @@ MCOUNT_LABEL(btrap) /* Traps that we leave interrupts disabled for. */ .macro TRAP_NOEN l, trapno - PTI_ENTRY \l,X\l + PTI_ENTRY \l,\l\()_pti_k,\l\()_pti_u +\l\()_pti_k: + subq $TF_RIP,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + movq $0,TF_ERR(%rsp) + jmp alltraps_noen_k +\l\()_pti_u: + subq $TF_RIP,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + movq $0,TF_ERR(%rsp) + jmp alltraps_noen_u + .globl X\l .type X\l,@function -X\l: subq $TF_RIP,%rsp - movl $\trapno,TF_TRAPNO(%rsp) - movq $0,TF_ADDR(%rsp) - movq $0,TF_ERR(%rsp) - jmp alltraps_noen +X\l: + subq $TF_RIP,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + movq $0,TF_ERR(%rsp) + testb $SEL_RPL_MASK,TF_CS(%rsp) + jz alltraps_noen_k + swapgs + jmp alltraps_noen_u .endm TRAP_NOEN bpt, T_BPTFLT @@ -122,15 +139,31 @@ X\l: subq $TF_RIP,%rsp /* Regular traps; The cpu does not supply tf_err for these. */ .macro TRAP l, trapno - PTI_ENTRY \l,X\l + PTI_ENTRY \l,\l\()_pti_k,\l\()_pti_u +\l\()_pti_k: + subq $TF_RIP,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + movq $0,TF_ERR(%rsp) + jmp alltraps_k +\l\()_pti_u: + subq $TF_RIP,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + movq $0,TF_ERR(%rsp) + jmp alltraps_u + .globl X\l .type X\l,@function X\l: - subq $TF_RIP,%rsp - movl $\trapno,TF_TRAPNO(%rsp) - movq $0,TF_ADDR(%rsp) - movq $0,TF_ERR(%rsp) - jmp alltraps + subq $TF_RIP,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + movq $0,TF_ERR(%rsp) + testb $SEL_RPL_MASK,TF_CS(%rsp) + jz alltraps_k + swapgs + jmp alltraps_u .endm TRAP div, T_DIVIDE @@ -145,42 +178,62 @@ X\l: /* This group of traps have tf_err already pushed by the cpu. */ .macro TRAP_ERR l, trapno - PTI_ENTRY \l,X\l,has_err=1 + PTI_ENTRY \l,\l\()_pti_k,\l\()_pti_u,has_err=1 +\l\()_pti_k: + subq $TF_ERR,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + jmp alltraps_k +\l\()_pti_u: + subq $TF_ERR,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + jmp alltraps_u .globl X\l .type X\l,@function X\l: - subq $TF_ERR,%rsp - movl $\trapno,TF_TRAPNO(%rsp) - movq $0,TF_ADDR(%rsp) - jmp alltraps + subq $TF_ERR,%rsp + movl $\trapno,TF_TRAPNO(%rsp) + movq $0,TF_ADDR(%rsp) + testb $SEL_RPL_MASK,TF_CS(%rsp) + jz alltraps_k + swapgs + jmp alltraps_u .endm TRAP_ERR tss, T_TSSFLT TRAP_ERR align, T_ALIGNFLT /* - * alltraps entry point. Use swapgs if this is the first time in the - * kernel from userland. Reenable interrupts if they were enabled - * before the trap. This approximates SDT_SYS386TGT on the i386 port. + * alltraps_u/k entry points. + * SWAPGS must be already performed by prologue, + * if this is the first time in the kernel from userland. + * Reenable interrupts if they were enabled before the trap. + * This approximates SDT_SYS386TGT on the i386 port. */ SUPERALIGN_TEXT - .globl alltraps - .type alltraps,@function -alltraps: + .globl alltraps_u + .type alltraps_u,@function +alltraps_u: movq %rdi,TF_RDI(%rsp) - testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ - jz 1f /* already running with kernel GS.base */ - swapgs + movq %rdx,TF_RDX(%rsp) + movq %rax,TF_RAX(%rsp) + movq %rcx,TF_RCX(%rsp) movq PCPU(CURPCB),%rdi andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) -1: SAVE_SEGS + call handle_ibrs_entry + jmp alltraps_save_segs + SUPERALIGN_TEXT + .globl alltraps_k + .type alltraps_k,@function +alltraps_k: + movq %rdi,TF_RDI(%rsp) movq %rdx,TF_RDX(%rsp) movq %rax,TF_RAX(%rsp) movq %rcx,TF_RCX(%rsp) - testb $SEL_RPL_MASK,TF_CS(%rsp) - jz 2f - call handle_ibrs_entry -2: testl $PSL_I,TF_RFLAGS(%rsp) +alltraps_save_segs: + SAVE_SEGS + testl $PSL_I,TF_RFLAGS(%rsp) jz alltraps_pushregs_no_rax sti alltraps_pushregs_no_rax: @@ -234,21 +287,26 @@ calltrap: jmp doreti /* Handle any pending ASTs */ /* - * alltraps_noen entry point. Unlike alltraps above, we want to - * leave the interrupts disabled. This corresponds to - * SDT_SYS386IGT on the i386 port. + * alltraps_noen_u/k entry points. + * Again, SWAPGS must be already performed by prologue, if needed. + * Unlike alltraps above, we want to leave the interrupts disabled. + * This corresponds to SDT_SYS386IGT on the i386 port. */ SUPERALIGN_TEXT - .globl alltraps_noen - .type alltraps_noen,@function -alltraps_noen: + .globl alltraps_noen_u + .type alltraps_noen_u,@function +alltraps_noen_u: movq %rdi,TF_RDI(%rsp) - testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ - jz 1f /* already running with kernel GS.base */ - swapgs movq PCPU(CURPCB),%rdi andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) -1: SAVE_SEGS + jmp alltraps_noen_save_segs + SUPERALIGN_TEXT + .globl alltraps_noen_k + .type alltraps_noen_k,@function +alltraps_noen_k: + movq %rdi,TF_RDI(%rsp) +alltraps_noen_save_segs: + SAVE_SEGS movq %rdx,TF_RDX(%rsp) movq %rax,TF_RAX(%rsp) movq %rcx,TF_RCX(%rsp) @@ -297,8 +355,8 @@ IDTVEC(dblfault) ALIGN_TEXT IDTVEC(page_pti) - testb $SEL_RPL_MASK,PTI_CS-2*8(%rsp) - jz Xpage + testb $SEL_RPL_MASK,PTI_CS-PTI_ERR(%rsp) + jz page_k swapgs pushq %rax movq %cr3,%rax @@ -306,25 +364,31 @@ IDTVEC(page_pti) cmpq $~0,PCPU(UCR3) jne 1f popq %rax - jmp 2f + jmp page_u 1: pushq %rdx PTI_UUENTRY has_err=1 -2: subq $TF_ERR,%rsp - movq %rdi,TF_RDI(%rsp) - movq %rax,TF_RAX(%rsp) - movq %rdx,TF_RDX(%rsp) - movq %rcx,TF_RCX(%rsp) jmp page_u + ALIGN_TEXT IDTVEC(page) + testb $SEL_RPL_MASK,TF_CS-TF_ERR(%rsp) /* Did we come from kernel? */ + jnz page_u_swapgs /* already running with kernel GS.base */ +page_k: subq $TF_ERR,%rsp movq %rdi,TF_RDI(%rsp) /* free up GP registers */ movq %rax,TF_RAX(%rsp) movq %rdx,TF_RDX(%rsp) movq %rcx,TF_RCX(%rsp) - testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ - jz page_cr2 /* already running with kernel GS.base */ + jmp page_cr2 + ALIGN_TEXT +page_u_swapgs: swapgs -page_u: movq PCPU(CURPCB),%rdi +page_u: + subq $TF_ERR,%rsp + movq %rdi,TF_RDI(%rsp) + movq %rax,TF_RAX(%rsp) + movq %rdx,TF_RDX(%rsp) + movq %rcx,TF_RCX(%rsp) + movq PCPU(CURPCB),%rdi andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) movq PCPU(SAVED_UCR3),%rax movq %rax,PCB_SAVED_UCR3(%rdi) Modified: head/sys/amd64/include/asmacros.h ============================================================================== --- head/sys/amd64/include/asmacros.h Sat Aug 3 16:57:14 2019 (r350560) +++ head/sys/amd64/include/asmacros.h Sat Aug 3 17:07:04 2019 (r350561) @@ -204,17 +204,16 @@ 1: .endm - .macro PTI_ENTRY name, cont, has_err=0 + .macro PTI_ENTRY name, contk, contu, has_err=0 ALIGN_TEXT .globl X\name\()_pti .type X\name\()_pti,@function X\name\()_pti: - /* %rax, %rdx and possibly err not yet pushed */ - testb $SEL_RPL_MASK,PTI_CS-(2+1-\has_err)*8(%rsp) - jz \cont + /* %rax, %rdx, and possibly err are not yet pushed */ + testb $SEL_RPL_MASK,PTI_CS-PTI_ERR-((1-\has_err)*8)(%rsp) + jz \contk PTI_UENTRY \has_err - swapgs - jmp \cont + jmp \contu .endm .macro PTI_INTRENTRY vec_name