Skip site navigation (1)Skip section navigation (2)
Date:      Fri, 22 Jun 2018 23:52:20 +0000 (UTC)
From:      Konstantin Belousov <kib@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-11@freebsd.org
Subject:   svn commit: r335570 - in stable/11/sys/amd64: amd64 include
Message-ID:  <201806222352.w5MNqKJg084967@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: kib
Date: Fri Jun 22 23:52:20 2018
New Revision: 335570
URL: https://svnweb.freebsd.org/changeset/base/335570

Log:
  MFC r333059 (by tychon):
  Expand the checks for UCR3 == PMAP_NO_CR3 to enable processes to be
  excluded from PTI.

Modified:
  stable/11/sys/amd64/amd64/cpu_switch.S
  stable/11/sys/amd64/amd64/exception.S
  stable/11/sys/amd64/amd64/genassym.c
  stable/11/sys/amd64/amd64/machdep.c
  stable/11/sys/amd64/amd64/mp_machdep.c
  stable/11/sys/amd64/amd64/pmap.c
  stable/11/sys/amd64/amd64/trap.c
  stable/11/sys/amd64/include/asmacros.h
  stable/11/sys/amd64/include/pcpu.h
Directory Properties:
  stable/11/   (props changed)

Modified: stable/11/sys/amd64/amd64/cpu_switch.S
==============================================================================
--- stable/11/sys/amd64/amd64/cpu_switch.S	Fri Jun 22 21:59:30 2018	(r335569)
+++ stable/11/sys/amd64/amd64/cpu_switch.S	Fri Jun 22 23:52:20 2018	(r335570)
@@ -210,10 +210,13 @@ done_tss:
 	movq	%r8,PCPU(RSP0)
 	movq	%r8,PCPU(CURPCB)
 	/* Update the TSS_RSP0 pointer for the next interrupt */
-	cmpb	$0,pti(%rip)
-	jne	1f
-	movq	%r8,TSS_RSP0(%rdx)
-1:	movq	%r12,PCPU(CURTHREAD)		/* into next thread */
+	cmpq	$~0,PCPU(UCR3)
+	je	1f
+	movq	PCPU(PTI_RSP0),%rax
+	movq	%rax,TSS_RSP0(%rdx)
+	jmp	2f
+1:	movq	%r8,TSS_RSP0(%rdx)
+2:	movq	%r12,PCPU(CURTHREAD)		/* into next thread */
 
 	/* Test if debug registers should be restored. */
 	testl	$PCB_DBREGS,PCB_FLAGS(%r8)
@@ -292,12 +295,7 @@ do_tss:	movq	%rdx,PCPU(TSSP)
 	shrq	$8,%rcx
 	movl	%ecx,8(%rax)
 	movb	$0x89,5(%rax)	/* unset busy */
-	cmpb	$0,pti(%rip)
-	je	1f
-	movq	PCPU(PRVSPACE),%rax
-	addq	$PC_PTI_STACK+PC_PTI_STACK_SZ*8,%rax
-	movq	%rax,TSS_RSP0(%rdx)
-1:	movl	$TSSSEL,%eax
+	movl	$TSSSEL,%eax
 	ltr	%ax
 	jmp	done_tss
 

Modified: stable/11/sys/amd64/amd64/exception.S
==============================================================================
--- stable/11/sys/amd64/amd64/exception.S	Fri Jun 22 21:59:30 2018	(r335569)
+++ stable/11/sys/amd64/amd64/exception.S	Fri Jun 22 23:52:20 2018	(r335570)
@@ -298,11 +298,15 @@ IDTVEC(page_pti)
 	jz	Xpage
 	swapgs
 	pushq	%rax
-	pushq	%rdx
 	movq	%cr3,%rax
 	movq	%rax,PCPU(SAVED_UCR3)
+	cmpq	$~0,PCPU(UCR3)
+	jne	1f
+	popq	%rax
+	jmp	2f
+1:	pushq	%rdx
 	PTI_UUENTRY has_err=1
-	subq	$TF_ERR,%rsp
+2:	subq	$TF_ERR,%rsp
 	movq	%rdi,TF_RDI(%rsp)
 	movq	%rax,TF_RAX(%rsp)
 	movq	%rdx,TF_RDX(%rsp)
@@ -347,9 +351,11 @@ page_cr2:
 	 */
 	.macro PROTF_ENTRY name,trapno
 \name\()_pti_doreti:
+	swapgs
+	cmpq	$~0,PCPU(UCR3)
+	je	1f
 	pushq	%rax
 	pushq	%rdx
-	swapgs
 	movq	PCPU(KCR3),%rax
 	movq	%rax,%cr3
 	movq	PCPU(RSP0),%rax
@@ -362,7 +368,7 @@ page_cr2:
 	movq	%rax,%rsp
 	popq	%rdx
 	popq	%rax
-	swapgs
+1:	swapgs
 	jmp	X\name
 IDTVEC(\name\()_pti)
 	cmpq	$doreti_iret,PTI_RIP-2*8(%rsp)
@@ -438,6 +444,8 @@ prot_addrf:
 IDTVEC(fast_syscall_pti)
 	swapgs
 	movq	%rax,PCPU(SCRATCH_RAX)
+	cmpq	$~0,PCPU(UCR3)
+	je	fast_syscall_common
 	movq	PCPU(KCR3),%rax
 	movq	%rax,%cr3
 	jmp	fast_syscall_common
@@ -503,7 +511,7 @@ fast_syscall_common:
 	movq	TF_RFLAGS(%rsp),%r11	/* original %rflags */
 	movq	TF_RIP(%rsp),%rcx	/* original %rip */
 	movq	TF_RSP(%rsp),%rsp	/* user stack pointer */
-	cmpb	$0,pti
+	cmpq	$~0,PCPU(UCR3)
 	je	2f
 	movq	PCPU(UCR3),%r9
 	movq	%r9,%cr3
@@ -1126,11 +1134,11 @@ ld_regs:
 	jz	2f			/* keep running with kernel GS.base */
 	cli
 	call	handle_ibrs_exit_rs
-	cmpb	$0,pti
+	cmpq	$~0,PCPU(UCR3)
 	je	1f
 	pushq	%rdx
-	movq	PCPU(PRVSPACE),%rdx
-	addq	$PC_PTI_STACK+PC_PTI_STACK_SZ*8-PTI_SIZE,%rdx
+	movq	PCPU(PTI_RSP0),%rdx
+	subq	$PTI_SIZE,%rdx
 	movq	%rax,PTI_RAX(%rdx)
 	popq	%rax
 	movq	%rax,PTI_RDX(%rdx)

Modified: stable/11/sys/amd64/amd64/genassym.c
==============================================================================
--- stable/11/sys/amd64/amd64/genassym.c	Fri Jun 22 21:59:30 2018	(r335569)
+++ stable/11/sys/amd64/amd64/genassym.c	Fri Jun 22 23:52:20 2018	(r335570)
@@ -229,6 +229,7 @@ ASSYM(PC_UCR3, offsetof(struct pcpu, pc_ucr3));
 ASSYM(PC_SAVED_UCR3, offsetof(struct pcpu, pc_saved_ucr3));
 ASSYM(PC_PTI_STACK, offsetof(struct pcpu, pc_pti_stack));
 ASSYM(PC_PTI_STACK_SZ, PC_PTI_STACK_SZ);
+ASSYM(PC_PTI_RSP0, offsetof(struct pcpu, pc_pti_rsp0));
 ASSYM(PC_IBPB_SET, offsetof(struct pcpu, pc_ibpb_set));
  
 ASSYM(LA_EOI, LAPIC_EOI * LAPIC_MEM_MUL);

Modified: stable/11/sys/amd64/amd64/machdep.c
==============================================================================
--- stable/11/sys/amd64/amd64/machdep.c	Fri Jun 22 21:59:30 2018	(r335569)
+++ stable/11/sys/amd64/amd64/machdep.c	Fri Jun 22 23:52:20 2018	(r335570)
@@ -1817,9 +1817,10 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	rsp0 = (vm_offset_t)thread0.td_pcb;
 	/* Ensure the stack is aligned to 16 bytes */
 	rsp0 &= ~0xFul;
-	common_tss[0].tss_rsp0 = pti ? ((vm_offset_t)PCPU_PTR(pti_stack) +
-	    PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful : rsp0;
+	common_tss[0].tss_rsp0 = rsp0;
 	PCPU_SET(rsp0, rsp0);
+	PCPU_SET(pti_rsp0, ((vm_offset_t)PCPU_PTR(pti_stack) +
+	    PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful);
 	PCPU_SET(curpcb, thread0.td_pcb);
 
 	/* transfer to user mode */

Modified: stable/11/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- stable/11/sys/amd64/amd64/mp_machdep.c	Fri Jun 22 21:59:30 2018	(r335569)
+++ stable/11/sys/amd64/amd64/mp_machdep.c	Fri Jun 22 23:52:20 2018	(r335570)
@@ -255,6 +255,8 @@ init_secondary(void)
 	pc->pc_tssp = &common_tss[cpu];
 	pc->pc_commontssp = &common_tss[cpu];
 	pc->pc_rsp0 = 0;
+	pc->pc_pti_rsp0 = ((vm_offset_t)&pc->pc_pti_stack +
+	    PC_PTI_STACK_SZ * sizeof(uint64_t) & ~0xful);
 	pc->pc_tss = (struct system_segment_descriptor *)&gdt[NGDT * cpu +
 	    GPROC0_SEL];
 	pc->pc_fs32p = &gdt[NGDT * cpu + GUFS32_SEL];
@@ -264,8 +266,7 @@ init_secondary(void)
 	pc->pc_curpmap = kernel_pmap;
 	pc->pc_pcid_gen = 1;
 	pc->pc_pcid_next = PMAP_PCID_KERN + 1;
-	common_tss[cpu].tss_rsp0 = pti ? ((vm_offset_t)&pc->pc_pti_stack +
-	    PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful : 0;
+	common_tss[cpu].tss_rsp0 = 0;
 
 	/* Save the per-cpu pointer for use by the NMI handler. */
 	np = ((struct nmi_pcpu *) &nmi_stack[PAGE_SIZE]) - 1;

Modified: stable/11/sys/amd64/amd64/pmap.c
==============================================================================
--- stable/11/sys/amd64/amd64/pmap.c	Fri Jun 22 21:59:30 2018	(r335569)
+++ stable/11/sys/amd64/amd64/pmap.c	Fri Jun 22 23:52:20 2018	(r335570)
@@ -2560,8 +2560,10 @@ pmap_pinit0(pmap_t pmap)
 	CPU_FOREACH(i) {
 		pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE;
 		pmap->pm_pcids[i].pm_gen = 0;
-		if (!pti)
+		if (!pti) {
 			__pcpu[i].pc_kcr3 = PMAP_NO_CR3;
+			__pcpu[i].pc_ucr3 = PMAP_NO_CR3;
+		}
 	}
 	PCPU_SET(curpmap, kernel_pmap);
 	pmap_activate(curthread);
@@ -2736,7 +2738,8 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, str
 			 * the kernel-mode page table active on return
 			 * to user space.
 			 */
-			*pml4 |= pg_nx;
+			if (pmap->pm_ucr3 != PMAP_NO_CR3)
+				*pml4 |= pg_nx;
 
 			pml4u = &pmap->pm_pml4u[pml4index];
 			*pml4u = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V |
@@ -7337,9 +7340,10 @@ pmap_activate_sw(struct thread *td)
 {
 	pmap_t oldpmap, pmap;
 	struct invpcid_descr d;
-	uint64_t cached, cr3, kcr3, kern_pti_cached, ucr3;
+	uint64_t cached, cr3, kcr3, kern_pti_cached, rsp0, ucr3;
 	register_t rflags;
 	u_int cpuid;
+	struct amd64tss *tssp;
 
 	oldpmap = PCPU_GET(curpmap);
 	pmap = vmspace_pmap(td->td_proc->p_vmspace);
@@ -7429,6 +7433,12 @@ pmap_activate_sw(struct thread *td)
 			PCPU_SET(kcr3, pmap->pm_cr3);
 			PCPU_SET(ucr3, pmap->pm_ucr3);
 		}
+	}
+	if (pmap->pm_ucr3 != PMAP_NO_CR3) {
+		rsp0 = ((vm_offset_t)PCPU_PTR(pti_stack) +
+		    PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful;
+		tssp = PCPU_GET(tssp);
+		tssp->tss_rsp0 = rsp0;
 	}
 #ifdef SMP
 	CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active);

Modified: stable/11/sys/amd64/amd64/trap.c
==============================================================================
--- stable/11/sys/amd64/amd64/trap.c	Fri Jun 22 21:59:30 2018	(r335569)
+++ stable/11/sys/amd64/amd64/trap.c	Fri Jun 22 23:52:20 2018	(r335570)
@@ -463,11 +463,13 @@ trap(struct trapframe *frame)
 			 */
 			if (frame->tf_rip == (long)doreti_iret) {
 				frame->tf_rip = (long)doreti_iret_fault;
-				if (pti && frame->tf_rsp == (uintptr_t)PCPU_PTR(
-				    pti_stack) + (PC_PTI_STACK_SZ - 5) *
-				    sizeof(register_t))
+				if ((PCPU_GET(curpmap)->pm_ucr3 !=
+				    PMAP_NO_CR3) &&
+				    (frame->tf_rsp == (uintptr_t)PCPU_GET(
+				    pti_rsp0) - 5 * sizeof(register_t))) {
 					frame->tf_rsp = PCPU_GET(rsp0) - 5 *
 					    sizeof(register_t);
+				}
 				return;
 			}
 			if (frame->tf_rip == (long)ld_ds) {

Modified: stable/11/sys/amd64/include/asmacros.h
==============================================================================
--- stable/11/sys/amd64/include/asmacros.h	Fri Jun 22 21:59:30 2018	(r335569)
+++ stable/11/sys/amd64/include/asmacros.h	Fri Jun 22 23:52:20 2018	(r335570)
@@ -194,9 +194,12 @@
 
 	.macro	PTI_UENTRY has_err
 	swapgs
+	cmpq	$~0,PCPU(UCR3)
+	je	1f
 	pushq	%rax
 	pushq	%rdx
 	PTI_UUENTRY \has_err
+1:
 	.endm
 
 	.macro	PTI_ENTRY name, cont, has_err=0

Modified: stable/11/sys/amd64/include/pcpu.h
==============================================================================
--- stable/11/sys/amd64/include/pcpu.h	Fri Jun 22 21:59:30 2018	(r335569)
+++ stable/11/sys/amd64/include/pcpu.h	Fri Jun 22 23:52:20 2018	(r335570)
@@ -67,13 +67,14 @@
 	u_int	pc_cmci_mask;		/* MCx banks for CMCI */	\
 	uint64_t pc_dbreg[16];		/* ddb debugging regs */	\
 	uint64_t pc_pti_stack[PC_PTI_STACK_SZ];				\
+	register_t pc_pti_rsp0;						\
 	int pc_dbreg_cmd;		/* ddb debugging reg cmd */	\
 	u_int	pc_vcpu_id;		/* Xen vCPU ID */		\
 	uint32_t pc_pcid_next;						\
 	uint32_t pc_pcid_gen;						\
 	uint32_t pc_smp_tlb_done;	/* TLB op acknowledgement */	\
 	uint32_t pc_ibpb_set;						\
-	char	__pad[96]		/* be divisor of PAGE_SIZE	\
+	char	__pad[88]		/* be divisor of PAGE_SIZE	\
 					   after cache alignment */
 
 #define	PC_DBREG_CMD_NONE	0



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201806222352.w5MNqKJg084967>