Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 19 Jul 2012 19:09:13 +0000 (UTC)
From:      Konstantin Belousov <kib@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r238623 - in head/sys/amd64: amd64 include
Message-ID:  <201207191909.q6JJ9D6Z011568@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: kib
Date: Thu Jul 19 19:09:12 2012
New Revision: 238623
URL: http://svn.freebsd.org/changeset/base/238623

Log:
  Introduce curpcb magic variable, similar to curthread, which is MD
  amd64.  It is implemented as __pure2 inline with non-volatile asm read
  from pcpu, which allows a compiler to cache its results.
  
  Convert most PCPU_GET(pcb) and curthread->td_pcb accesses into curpcb.
  
  Note that __curthread() uses magic value 0 as an offsetof(struct pcpu,
  pc_curthread). It seems to be done this way due to machine/pcpu.h
  needs to be processed before sys/pcpu.h, because machine/pcpu.h
  contributes machine-depended fields to the struct pcpu definition. As
  result, machine/pcpu.h cannot use struct pcpu yet.
  
  The __curpcb() also uses a magic constant instead of offsetof(struct
  pcpu, pc_curpcb) for the same reason. The constants are now defined as
  symbols and CTASSERTs are added to ensure that future KBI changes do
  not break the code.
  
  Requested and reviewed by: bde
  MFC after:    3 weeks

Modified:
  head/sys/amd64/amd64/fpu.c
  head/sys/amd64/amd64/machdep.c
  head/sys/amd64/amd64/trap.c
  head/sys/amd64/amd64/vm_machdep.c
  head/sys/amd64/include/pcpu.h

Modified: head/sys/amd64/amd64/fpu.c
==============================================================================
--- head/sys/amd64/amd64/fpu.c	Thu Jul 19 15:36:36 2012	(r238622)
+++ head/sys/amd64/amd64/fpu.c	Thu Jul 19 19:09:12 2012	(r238623)
@@ -327,7 +327,7 @@ fpuexit(struct thread *td)
 	critical_enter();
 	if (curthread == PCPU_GET(fpcurthread)) {
 		stop_emulating();
-		fpusave(PCPU_GET(curpcb)->pcb_save);
+		fpusave(curpcb->pcb_save);
 		start_emulating();
 		PCPU_SET(fpcurthread, 0);
 	}
@@ -547,7 +547,7 @@ fputrap_x87(void)
 	 * wherever they are.
 	 */
 	if (PCPU_GET(fpcurthread) != curthread) {
-		pcb_save = PCPU_GET(curpcb)->pcb_save;
+		pcb_save = curpcb->pcb_save;
 		control = pcb_save->sv_env.en_cw;
 		status = pcb_save->sv_env.en_sw;
 	} else {
@@ -567,7 +567,7 @@ fputrap_sse(void)
 
 	critical_enter();
 	if (PCPU_GET(fpcurthread) != curthread)
-		mxcsr = PCPU_GET(curpcb)->pcb_save->sv_env.en_mxcsr;
+		mxcsr = curpcb->pcb_save->sv_env.en_mxcsr;
 	else
 		stmxcsr(&mxcsr);
 	critical_exit();
@@ -609,7 +609,7 @@ fpudna(void)
 	 * Record new context early in case frstor causes a trap.
 	 */
 	PCPU_SET(fpcurthread, curthread);
-	pcb = PCPU_GET(curpcb);
+	pcb = curpcb;
 
 	fpu_clean_state();
 
@@ -970,7 +970,7 @@ fpu_kern_thread(u_int flags)
 {
 	struct pcb *pcb;
 
-	pcb = PCPU_GET(curpcb);
+	pcb = curpcb;
 	KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0,
 	    ("Only kthread may use fpu_kern_thread"));
 	KASSERT(pcb->pcb_save == get_pcb_user_save_pcb(pcb),
@@ -987,5 +987,5 @@ is_fpu_kern_thread(u_int flags)
 
 	if ((curthread->td_pflags & TDP_KTHREAD) == 0)
 		return (0);
-	return ((PCPU_GET(curpcb)->pcb_flags & PCB_KERNFPU) != 0);
+	return ((curpcb->pcb_flags & PCB_KERNFPU) != 0);
 }

Modified: head/sys/amd64/amd64/machdep.c
==============================================================================
--- head/sys/amd64/amd64/machdep.c	Thu Jul 19 15:36:36 2012	(r238622)
+++ head/sys/amd64/amd64/machdep.c	Thu Jul 19 19:09:12 2012	(r238623)
@@ -996,7 +996,7 @@ exec_setregs(struct thread *td, struct i
 		pcb->pcb_dr3 = 0;
 		pcb->pcb_dr6 = 0;
 		pcb->pcb_dr7 = 0;
-		if (pcb == PCPU_GET(curpcb)) {
+		if (pcb == curpcb) {
 			/*
 			 * Clear the debug registers on the running
 			 * CPU, otherwise they will end up affecting

Modified: head/sys/amd64/amd64/trap.c
==============================================================================
--- head/sys/amd64/amd64/trap.c	Thu Jul 19 15:36:36 2012	(r238622)
+++ head/sys/amd64/amd64/trap.c	Thu Jul 19 19:09:12 2012	(r238623)
@@ -520,9 +520,8 @@ trap(struct trapframe *frame)
 				frame->tf_rip = (long)fsbase_load_fault;
 				goto out;
 			}
-			if (PCPU_GET(curpcb)->pcb_onfault != NULL) {
-				frame->tf_rip =
-				    (long)PCPU_GET(curpcb)->pcb_onfault;
+			if (curpcb->pcb_onfault != NULL) {
+				frame->tf_rip = (long)curpcb->pcb_onfault;
 				goto out;
 			}
 			break;
@@ -708,7 +707,7 @@ trap_pfault(frame, usermode)
 		 * it normally, and panic immediately.
 		 */
 		if (!usermode && (td->td_intr_nesting_level != 0 ||
-		    PCPU_GET(curpcb)->pcb_onfault == NULL)) {
+		    curpcb->pcb_onfault == NULL)) {
 			trap_fatal(frame, eva);
 			return (-1);
 		}
@@ -764,8 +763,8 @@ trap_pfault(frame, usermode)
 nogo:
 	if (!usermode) {
 		if (td->td_intr_nesting_level == 0 &&
-		    PCPU_GET(curpcb)->pcb_onfault != NULL) {
-			frame->tf_rip = (long)PCPU_GET(curpcb)->pcb_onfault;
+		    curpcb->pcb_onfault != NULL) {
+			frame->tf_rip = (long)curpcb->pcb_onfault;
 			return (0);
 		}
 		trap_fatal(frame, eva);

Modified: head/sys/amd64/amd64/vm_machdep.c
==============================================================================
--- head/sys/amd64/amd64/vm_machdep.c	Thu Jul 19 15:36:36 2012	(r238622)
+++ head/sys/amd64/amd64/vm_machdep.c	Thu Jul 19 19:09:12 2012	(r238623)
@@ -90,6 +90,10 @@ static u_int	cpu_reset_proxyid;
 static volatile u_int	cpu_reset_proxy_active;
 #endif
 
+CTASSERT((struct thread **)OFFSETOF_CURTHREAD ==
+    &((struct pcpu *)NULL)->pc_curthread);
+CTASSERT((struct pcb **)OFFSETOF_CURPCB == &((struct pcpu *)NULL)->pc_curpcb);
+
 struct savefpu *
 get_pcb_user_save_td(struct thread *td)
 {

Modified: head/sys/amd64/include/pcpu.h
==============================================================================
--- head/sys/amd64/include/pcpu.h	Thu Jul 19 15:36:36 2012	(r238622)
+++ head/sys/amd64/include/pcpu.h	Thu Jul 19 19:09:12 2012	(r238623)
@@ -216,16 +216,29 @@ extern struct pcpu *pcpup;
 #define	PCPU_PTR(member)	__PCPU_PTR(pc_ ## member)
 #define	PCPU_SET(member, val)	__PCPU_SET(pc_ ## member, val)
 
+#define	OFFSETOF_CURTHREAD	0
 static __inline __pure2 struct thread *
 __curthread(void)
 {
 	struct thread *td;
 
-	__asm("movq %%gs:0,%0" : "=r" (td));
+	__asm("movq %%gs:%1,%0" : "=r" (td)
+	    : "m" (*(char *)OFFSETOF_CURTHREAD));
 	return (td);
 }
 #define	curthread		(__curthread())
 
+#define	OFFSETOF_CURPCB		32
+static __inline __pure2 struct pcb *
+__curpcb(void)
+{
+	struct pcb *pcb;
+
+	__asm("movq %%gs:%1,%0" : "=r" (pcb) : "m" (*(char *)OFFSETOF_CURPCB));
+	return (pcb);
+}
+#define	curpcb		(__curpcb())
+
 #define	IS_BSP()	(PCPU_GET(cpuid) == 0)
 
 #else /* !lint || defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE___TYPEOF) */



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201207191909.q6JJ9D6Z011568>