Skip site navigation (1)Skip section navigation (2)
Date:      Sat, 30 Oct 2010 23:07:30 +0000 (UTC)
From:      Nathan Whitehorn <nwhitehorn@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r214574 - in head/sys/powerpc: aim include powerpc
Message-ID:  <201010302307.o9UN7U2u029753@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: nwhitehorn
Date: Sat Oct 30 23:07:30 2010
New Revision: 214574
URL: http://svn.freebsd.org/changeset/base/214574

Log:
  Restructure the way the copyin/copyout segment is stored to prevent a
  concurrency bug. Since all SLB/SR entries were invalidated during an
  exception, a decrementer exception could cause the user segment to be
  invalidated during a copyin()/copyout() without a thread switch that
  would cause it to be restored from the PCB, potentially causing the
  operation to continue on invalid memory. This is now handled by explicit
  restoration of segment 12 from the PCB on 32-bit systems and a check in
  the Data Segment Exception handler on 64-bit.
  
  While here, cause copyin()/copyout() to check whether the requested
  user segment is already installed, saving some pipeline flushes, and
  fix the synchronization primitives around the mtsr and slbmte
  instructions to prevent accessing stale segments.
  
  MFC after:	2 weeks

Modified:
  head/sys/powerpc/aim/copyinout.c
  head/sys/powerpc/aim/slb.c
  head/sys/powerpc/aim/swtch64.S
  head/sys/powerpc/aim/trap.c
  head/sys/powerpc/aim/trap_subr32.S
  head/sys/powerpc/aim/trap_subr64.S
  head/sys/powerpc/aim/vm_machdep.c
  head/sys/powerpc/include/pcb.h
  head/sys/powerpc/include/slb.h
  head/sys/powerpc/include/sr.h
  head/sys/powerpc/powerpc/exec_machdep.c
  head/sys/powerpc/powerpc/genassym.c

Modified: head/sys/powerpc/aim/copyinout.c
==============================================================================
--- head/sys/powerpc/aim/copyinout.c	Sat Oct 30 23:04:54 2010	(r214573)
+++ head/sys/powerpc/aim/copyinout.c	Sat Oct 30 23:07:30 2010	(r214574)
@@ -81,9 +81,7 @@ static __inline void
 set_user_sr(pmap_t pm, const void *addr)
 {
 	struct slb *slb;
-	register_t esid, vsid, slb1, slb2;
-
-	esid = USER_ADDR >> ADDR_SR_SHFT;
+	register_t slbv;
 
 	/* Try lockless look-up first */
 	slb = user_va_to_slb_entry(pm, (vm_offset_t)addr);
@@ -91,20 +89,21 @@ set_user_sr(pmap_t pm, const void *addr)
 	if (slb == NULL) {
 		/* If it isn't there, we need to pre-fault the VSID */
 		PMAP_LOCK(pm);
-		vsid = va_to_vsid(pm, (vm_offset_t)addr);
+		slbv = va_to_vsid(pm, (vm_offset_t)addr) << SLBV_VSID_SHIFT;
 		PMAP_UNLOCK(pm);
 	} else {
-		vsid = slb->slbv >> SLBV_VSID_SHIFT;
+		slbv = slb->slbv;
 	}
 
-	slb1 = vsid << SLBV_VSID_SHIFT;
-	slb2 = (esid << SLBE_ESID_SHIFT) | SLBE_VALID | USER_SR;
+	/* If we have already set this VSID, we can just return */
+	if (curthread->td_pcb->pcb_cpu.aim.usr_vsid == slbv) 
+		return;
 
+	__asm __volatile ("isync; slbie %0; slbmte %1, %2; isync" ::
+	    "r"(USER_ADDR), "r"(slbv), "r"(USER_SLB_SLBE));
 	curthread->td_pcb->pcb_cpu.aim.usr_segm =
 	    (uintptr_t)addr >> ADDR_SR_SHFT;
-	__asm __volatile ("slbie %0; slbmte %1, %2" :: "r"(esid << 28),
-	    "r"(slb1), "r"(slb2));
-	isync();
+	curthread->td_pcb->pcb_cpu.aim.usr_vsid = slbv;
 }
 #else
 static __inline void
@@ -114,9 +113,13 @@ set_user_sr(pmap_t pm, const void *addr)
 
 	vsid = va_to_vsid(pm, (vm_offset_t)addr);
 
-	isync();
-	__asm __volatile ("mtsr %0,%1" :: "n"(USER_SR), "r"(vsid));
-	isync();
+	/* If we have already set this VSID, we can just return */
+	if (curthread->td_pcb->pcb_cpu.aim.usr_vsid == vsid)
+		return;
+
+	__asm __volatile ("sync; mtsr %0,%1; sync; isync" :: "n"(USER_SR),
+	    "r"(vsid));
+	curthread->td_pcb->pcb_cpu.aim.usr_vsid = vsid;
 }
 #endif
 

Modified: head/sys/powerpc/aim/slb.c
==============================================================================
--- head/sys/powerpc/aim/slb.c	Sat Oct 30 23:04:54 2010	(r214573)
+++ head/sys/powerpc/aim/slb.c	Sat Oct 30 23:07:30 2010	(r214574)
@@ -200,7 +200,7 @@ kernel_va_to_slbv(vm_offset_t va)
 	esid = (uintptr_t)va >> ADDR_SR_SHFT;
 
 	/* Set kernel VSID to deterministic value */
-	slbv = va_to_vsid(kernel_pmap, va) << SLBV_VSID_SHIFT;
+	slbv = (KERNEL_VSID((uintptr_t)va >> ADDR_SR_SHFT)) << SLBV_VSID_SHIFT;
 
 	/* Figure out if this is a large-page mapping */
 	if (hw_direct_map && va < VM_MIN_KERNEL_ADDRESS) {
@@ -421,19 +421,19 @@ slb_insert_kernel(uint64_t slbe, uint64_
 
 	slbcache = PCPU_GET(slb);
 
-	/* Check for an unused slot, abusing the USER_SR slot as a full flag */
-	if (slbcache[USER_SR].slbe == 0) {
-		for (i = 0; i < USER_SR; i++) {
+	/* Check for an unused slot, abusing the user slot as a full flag */
+	if (slbcache[USER_SLB_SLOT].slbe == 0) {
+		for (i = 0; i < USER_SLB_SLOT; i++) {
 			if (!(slbcache[i].slbe & SLBE_VALID)) 
 				goto fillkernslb;
 		}
 
-		if (i == USER_SR)
-			slbcache[USER_SR].slbe = 1;
+		if (i == USER_SLB_SLOT)
+			slbcache[USER_SLB_SLOT].slbe = 1;
 	}
 
 	for (i = mftb() % 64, j = 0; j < 64; j++, i = (i+1) % 64) {
-		if (i == USER_SR)
+		if (i == USER_SLB_SLOT)
 			continue;
 
 		if (SLB_SPILLABLE(slbcache[i].slbe))

Modified: head/sys/powerpc/aim/swtch64.S
==============================================================================
--- head/sys/powerpc/aim/swtch64.S	Sat Oct 30 23:04:54 2010	(r214573)
+++ head/sys/powerpc/aim/swtch64.S	Sat Oct 30 23:07:30 2010	(r214574)
@@ -110,13 +110,10 @@ ENTRY(cpu_switch)
 	std	%r1,PCB_SP(%r6)		/* Save the stack pointer */
 	std	%r2,PCB_TOC(%r6)	/* Save the TOC pointer */
 	
-	li	%r14,0			/* Save USER_SR for copyin/out */
-	li	%r15,0
-	li	%r16,USER_SR
-	slbmfee %r14, %r16
+	li	%r15,0			/* Save user segment for copyin/out */
+	li	%r16,USER_SLB_SLOT
 	slbmfev %r15, %r16
 	isync
-	std	%r14,PCB_AIM_USR_ESID(%r6)
 	std	%r15,PCB_AIM_USR_VSID(%r6)
 
 	mr	%r14,%r3		/* Copy the old thread ptr... */
@@ -221,14 +218,17 @@ blocked_loop:
 	ld	%r1,PCB_SP(%r3)		/* Load the stack pointer */
 	ld	%r2,PCB_TOC(%r3)	/* Load the TOC pointer */
 
-	lis	%r5,USER_ADDR@highesta	/* Load the USER_SR segment reg */
+	lis	%r5,USER_ADDR@highesta	/* Load the copyin/out segment reg */
 	ori	%r5,%r5,USER_ADDR@highera
 	sldi	%r5,%r5,32
 	oris	%r5,%r5,USER_ADDR@ha
 	slbie	%r5
+	lis	%r6,USER_SLB_SLBE@highesta
+	ori	%r6,%r6,USER_SLB_SLBE@highera
+	sldi	%r6,%r6,32
+	oris	%r6,%r6,USER_SLB_SLBE@ha
+	ori	%r6,%r6,USER_SLB_SLBE@l
 	ld	%r5,PCB_AIM_USR_VSID(%r3)
-	ld	%r6,PCB_AIM_USR_ESID(%r3)
-	ori	%r6,%r6,USER_SR
 	slbmte	%r5,%r6
 
 	isync

Modified: head/sys/powerpc/aim/trap.c
==============================================================================
--- head/sys/powerpc/aim/trap.c	Sat Oct 30 23:04:54 2010	(r214573)
+++ head/sys/powerpc/aim/trap.c	Sat Oct 30 23:07:30 2010	(r214574)
@@ -249,8 +249,16 @@ trap(struct trapframe *frame)
  				return;
 			break;
 #ifdef __powerpc64__
-		case EXC_ISE:
 		case EXC_DSE:
+			if ((frame->cpu.aim.dar & SEGMENT_MASK) == USER_ADDR) {
+				__asm __volatile ("slbmte %0, %1" ::
+				     "r"(td->td_pcb->pcb_cpu.aim.usr_vsid),
+				     "r"(USER_SLB_SLBE));
+				return;
+			}
+
+			/* FALLTHROUGH */
+		case EXC_ISE:
 			if (handle_slb_spill(kernel_pmap,
 			    (type == EXC_ISE) ? frame->srr0 :
 			    frame->cpu.aim.dar) != 0)

Modified: head/sys/powerpc/aim/trap_subr32.S
==============================================================================
--- head/sys/powerpc/aim/trap_subr32.S	Sat Oct 30 23:04:54 2010	(r214573)
+++ head/sys/powerpc/aim/trap_subr32.S	Sat Oct 30 23:07:30 2010	(r214574)
@@ -54,7 +54,7 @@
 	lwz	sr,9*4(pmap);	mtsr	9,sr; \
 	lwz	sr,10*4(pmap);	mtsr	10,sr; \
 	lwz	sr,11*4(pmap);	mtsr	11,sr; \
-	lwz	sr,12*4(pmap);	mtsr	12,sr; \
+	/* Skip segment 12 (USER_SR), which is restored differently */ \
 	lwz	sr,13*4(pmap);	mtsr	13,sr; \
 	lwz	sr,14*4(pmap);	mtsr	14,sr; \
 	lwz	sr,15*4(pmap);	mtsr	15,sr; isync;
@@ -66,7 +66,9 @@
 	GET_CPUINFO(pmap); \
 	lwz	pmap,PC_CURPMAP(pmap); \
 	lwzu	sr,PM_SR(pmap); \
-	RESTORE_SRS(pmap,sr)
+	RESTORE_SRS(pmap,sr) \
+	/* Restore SR 12 */ \
+	lwz	sr,12*4(pmap);	mtsr	12,sr
 
 /*
  * Kernel SRs are loaded directly from kernel_pmap_
@@ -537,6 +539,11 @@ u_trap:
  */
 k_trap:
 	FRAME_SETUP(PC_TEMPSAVE)
+	/* Restore USER_SR */
+	GET_CPUINFO(%r30)
+	lwz	%r30,PC_CURPCB(%r30)
+	lwz	%r30,PCB_AIM_USR_VSID(%r30)
+	mtsr	USER_SR,%r30; sync; isync
 /* Call C interrupt dispatcher: */
 trapagain:
 	addi	%r3,%r1,8

Modified: head/sys/powerpc/aim/trap_subr64.S
==============================================================================
--- head/sys/powerpc/aim/trap_subr64.S	Sat Oct 30 23:04:54 2010	(r214573)
+++ head/sys/powerpc/aim/trap_subr64.S	Sat Oct 30 23:07:30 2010	(r214574)
@@ -99,7 +99,7 @@ instkernslb:
 
 	addi	%r28, %r28, 16;		/* Advance pointer */
 	addi	%r29, %r29, 1;
-	cmpli	0, %r29, USER_SR;	/* Repeat if we are not at the end */
+	cmpli	0, %r29, USER_SLB_SLOT;	/* Repeat if we are not at the end */
 	blt instkernslb;
 	blr;
 

Modified: head/sys/powerpc/aim/vm_machdep.c
==============================================================================
--- head/sys/powerpc/aim/vm_machdep.c	Sat Oct 30 23:04:54 2010	(r214573)
+++ head/sys/powerpc/aim/vm_machdep.c	Sat Oct 30 23:07:30 2010	(r214574)
@@ -197,7 +197,6 @@ cpu_fork(struct thread *td1, struct proc
 	pcb->pcb_lr = (register_t)fork_trampoline;
 	#endif
 	pcb->pcb_cpu.aim.usr_vsid = 0;
-	pcb->pcb_cpu.aim.usr_esid = 0;
 
 	/* Setup to release spin count in fork_exit(). */
 	td2->td_md.md_spinlock_count = 1;

Modified: head/sys/powerpc/include/pcb.h
==============================================================================
--- head/sys/powerpc/include/pcb.h	Sat Oct 30 23:04:54 2010	(r214573)
+++ head/sys/powerpc/include/pcb.h	Sat Oct 30 23:07:30 2010	(r214574)
@@ -67,7 +67,6 @@ struct pcb {
 	union {
 		struct {
 			vm_offset_t	usr_segm;	/* Base address */
-			register_t	usr_esid;	/* USER_SR segment */
 			register_t	usr_vsid;	/* USER_SR segment */
 		} aim;
 		struct {

Modified: head/sys/powerpc/include/slb.h
==============================================================================
--- head/sys/powerpc/include/slb.h	Sat Oct 30 23:04:54 2010	(r214573)
+++ head/sys/powerpc/include/slb.h	Sat Oct 30 23:07:30 2010	(r214574)
@@ -62,6 +62,13 @@
 #define	SLBE_ESID_MASK	0xfffffffff0000000UL /* Effective segment ID mask */
 #define	SLBE_ESID_SHIFT	28
 
+/*
+ * User segment for copyin/out
+ */
+#define USER_SLB_SLOT 63
+#define USER_SLB_SLBE (((USER_ADDR >> ADDR_SR_SHFT) << SLBE_ESID_SHIFT) | \
+			SLBE_VALID | USER_SLB_SLOT)
+
 struct slb {
 	uint64_t	slbv;
 	uint64_t	slbe;

Modified: head/sys/powerpc/include/sr.h
==============================================================================
--- head/sys/powerpc/include/sr.h	Sat Oct 30 23:04:54 2010	(r214573)
+++ head/sys/powerpc/include/sr.h	Sat Oct 30 23:07:30 2010	(r214574)
@@ -42,11 +42,7 @@
 #define	SR_VSID_MASK	0x00ffffff	/* Virtual Segment ID mask */
 
 /* Kernel segment register usage */
-#ifdef __powerpc64__
-#define	USER_SR		63
-#else
 #define	USER_SR		12
-#endif
 #define	KERNEL_SR	13
 #define	KERNEL2_SR	14
 #define	KERNEL3_SR	15

Modified: head/sys/powerpc/powerpc/exec_machdep.c
==============================================================================
--- head/sys/powerpc/powerpc/exec_machdep.c	Sat Oct 30 23:04:54 2010	(r214573)
+++ head/sys/powerpc/powerpc/exec_machdep.c	Sat Oct 30 23:07:30 2010	(r214574)
@@ -986,7 +986,6 @@ cpu_set_upcall(struct thread *td, struct
 	pcb2->pcb_lr = (register_t)fork_trampoline;
 	#endif
 	pcb2->pcb_cpu.aim.usr_vsid = 0;
-	pcb2->pcb_cpu.aim.usr_esid = 0;
 
 	/* Setup to release spin count in fork_exit(). */
 	td->td_md.md_spinlock_count = 1;

Modified: head/sys/powerpc/powerpc/genassym.c
==============================================================================
--- head/sys/powerpc/powerpc/genassym.c	Sat Oct 30 23:04:54 2010	(r214573)
+++ head/sys/powerpc/powerpc/genassym.c	Sat Oct 30 23:07:30 2010	(r214574)
@@ -103,13 +103,15 @@ ASSYM(TLBSAVE_BOOKE_R31, TLBSAVE_BOOKE_R
 ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock));
 
 #if defined(AIM)
-ASSYM(USER_SR, USER_SR);
 ASSYM(USER_ADDR, USER_ADDR);
 #ifdef __powerpc64__
 ASSYM(PC_KERNSLB, offsetof(struct pcpu, pc_slb));
 ASSYM(PC_USERSLB, offsetof(struct pcpu, pc_userslb));
+ASSYM(USER_SLB_SLOT, USER_SLB_SLOT);
+ASSYM(USER_SLB_SLBE, USER_SLB_SLBE);
 #else
 ASSYM(PM_SR, offsetof(struct pmap, pm_sr));
+ASSYM(USER_SR, USER_SR);
 #endif
 #elif defined(E500)
 ASSYM(PM_PDIR, offsetof(struct pmap, pm_pdir));
@@ -187,7 +189,6 @@ ASSYM(PCB_FLAGS, offsetof(struct pcb, pc
 ASSYM(PCB_FPU, PCB_FPU);
 ASSYM(PCB_VEC, PCB_VEC);
 
-ASSYM(PCB_AIM_USR_ESID, offsetof(struct pcb, pcb_cpu.aim.usr_esid));
 ASSYM(PCB_AIM_USR_VSID, offsetof(struct pcb, pcb_cpu.aim.usr_vsid));
 ASSYM(PCB_BOOKE_CTR, offsetof(struct pcb, pcb_cpu.booke.ctr));
 ASSYM(PCB_BOOKE_XER, offsetof(struct pcb, pcb_cpu.booke.xer));



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201010302307.o9UN7U2u029753>