From owner-svn-src-all@FreeBSD.ORG Thu Jul 19 19:09:13 2012 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id AF1C21065673; Thu, 19 Jul 2012 19:09:13 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 995748FC0A; Thu, 19 Jul 2012 19:09:13 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.4/8.14.4) with ESMTP id q6JJ9DeO011574; Thu, 19 Jul 2012 19:09:13 GMT (envelope-from kib@svn.freebsd.org) Received: (from kib@localhost) by svn.freebsd.org (8.14.4/8.14.4/Submit) id q6JJ9D6Z011568; Thu, 19 Jul 2012 19:09:13 GMT (envelope-from kib@svn.freebsd.org) Message-Id: <201207191909.q6JJ9D6Z011568@svn.freebsd.org> From: Konstantin Belousov Date: Thu, 19 Jul 2012 19:09:13 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r238623 - in head/sys/amd64: amd64 include X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 19 Jul 2012 19:09:13 -0000 Author: kib Date: Thu Jul 19 19:09:12 2012 New Revision: 238623 URL: http://svn.freebsd.org/changeset/base/238623 Log: Introduce curpcb magic variable, similar to curthread, which is MD amd64. It is implemented as __pure2 inline with non-volatile asm read from pcpu, which allows a compiler to cache its results. Convert most PCPU_GET(pcb) and curthread->td_pcb accesses into curpcb. Note that __curthread() uses magic value 0 as an offsetof(struct pcpu, pc_curthread). It seems to be done this way due to machine/pcpu.h needs to be processed before sys/pcpu.h, because machine/pcpu.h contributes machine-depended fields to the struct pcpu definition. As result, machine/pcpu.h cannot use struct pcpu yet. The __curpcb() also uses a magic constant instead of offsetof(struct pcpu, pc_curpcb) for the same reason. The constants are now defined as symbols and CTASSERTs are added to ensure that future KBI changes do not break the code. Requested and reviewed by: bde MFC after: 3 weeks Modified: head/sys/amd64/amd64/fpu.c head/sys/amd64/amd64/machdep.c head/sys/amd64/amd64/trap.c head/sys/amd64/amd64/vm_machdep.c head/sys/amd64/include/pcpu.h Modified: head/sys/amd64/amd64/fpu.c ============================================================================== --- head/sys/amd64/amd64/fpu.c Thu Jul 19 15:36:36 2012 (r238622) +++ head/sys/amd64/amd64/fpu.c Thu Jul 19 19:09:12 2012 (r238623) @@ -327,7 +327,7 @@ fpuexit(struct thread *td) critical_enter(); if (curthread == PCPU_GET(fpcurthread)) { stop_emulating(); - fpusave(PCPU_GET(curpcb)->pcb_save); + fpusave(curpcb->pcb_save); start_emulating(); PCPU_SET(fpcurthread, 0); } @@ -547,7 +547,7 @@ fputrap_x87(void) * wherever they are. */ if (PCPU_GET(fpcurthread) != curthread) { - pcb_save = PCPU_GET(curpcb)->pcb_save; + pcb_save = curpcb->pcb_save; control = pcb_save->sv_env.en_cw; status = pcb_save->sv_env.en_sw; } else { @@ -567,7 +567,7 @@ fputrap_sse(void) critical_enter(); if (PCPU_GET(fpcurthread) != curthread) - mxcsr = PCPU_GET(curpcb)->pcb_save->sv_env.en_mxcsr; + mxcsr = curpcb->pcb_save->sv_env.en_mxcsr; else stmxcsr(&mxcsr); critical_exit(); @@ -609,7 +609,7 @@ fpudna(void) * Record new context early in case frstor causes a trap. */ PCPU_SET(fpcurthread, curthread); - pcb = PCPU_GET(curpcb); + pcb = curpcb; fpu_clean_state(); @@ -970,7 +970,7 @@ fpu_kern_thread(u_int flags) { struct pcb *pcb; - pcb = PCPU_GET(curpcb); + pcb = curpcb; KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0, ("Only kthread may use fpu_kern_thread")); KASSERT(pcb->pcb_save == get_pcb_user_save_pcb(pcb), @@ -987,5 +987,5 @@ is_fpu_kern_thread(u_int flags) if ((curthread->td_pflags & TDP_KTHREAD) == 0) return (0); - return ((PCPU_GET(curpcb)->pcb_flags & PCB_KERNFPU) != 0); + return ((curpcb->pcb_flags & PCB_KERNFPU) != 0); } Modified: head/sys/amd64/amd64/machdep.c ============================================================================== --- head/sys/amd64/amd64/machdep.c Thu Jul 19 15:36:36 2012 (r238622) +++ head/sys/amd64/amd64/machdep.c Thu Jul 19 19:09:12 2012 (r238623) @@ -996,7 +996,7 @@ exec_setregs(struct thread *td, struct i pcb->pcb_dr3 = 0; pcb->pcb_dr6 = 0; pcb->pcb_dr7 = 0; - if (pcb == PCPU_GET(curpcb)) { + if (pcb == curpcb) { /* * Clear the debug registers on the running * CPU, otherwise they will end up affecting Modified: head/sys/amd64/amd64/trap.c ============================================================================== --- head/sys/amd64/amd64/trap.c Thu Jul 19 15:36:36 2012 (r238622) +++ head/sys/amd64/amd64/trap.c Thu Jul 19 19:09:12 2012 (r238623) @@ -520,9 +520,8 @@ trap(struct trapframe *frame) frame->tf_rip = (long)fsbase_load_fault; goto out; } - if (PCPU_GET(curpcb)->pcb_onfault != NULL) { - frame->tf_rip = - (long)PCPU_GET(curpcb)->pcb_onfault; + if (curpcb->pcb_onfault != NULL) { + frame->tf_rip = (long)curpcb->pcb_onfault; goto out; } break; @@ -708,7 +707,7 @@ trap_pfault(frame, usermode) * it normally, and panic immediately. */ if (!usermode && (td->td_intr_nesting_level != 0 || - PCPU_GET(curpcb)->pcb_onfault == NULL)) { + curpcb->pcb_onfault == NULL)) { trap_fatal(frame, eva); return (-1); } @@ -764,8 +763,8 @@ trap_pfault(frame, usermode) nogo: if (!usermode) { if (td->td_intr_nesting_level == 0 && - PCPU_GET(curpcb)->pcb_onfault != NULL) { - frame->tf_rip = (long)PCPU_GET(curpcb)->pcb_onfault; + curpcb->pcb_onfault != NULL) { + frame->tf_rip = (long)curpcb->pcb_onfault; return (0); } trap_fatal(frame, eva); Modified: head/sys/amd64/amd64/vm_machdep.c ============================================================================== --- head/sys/amd64/amd64/vm_machdep.c Thu Jul 19 15:36:36 2012 (r238622) +++ head/sys/amd64/amd64/vm_machdep.c Thu Jul 19 19:09:12 2012 (r238623) @@ -90,6 +90,10 @@ static u_int cpu_reset_proxyid; static volatile u_int cpu_reset_proxy_active; #endif +CTASSERT((struct thread **)OFFSETOF_CURTHREAD == + &((struct pcpu *)NULL)->pc_curthread); +CTASSERT((struct pcb **)OFFSETOF_CURPCB == &((struct pcpu *)NULL)->pc_curpcb); + struct savefpu * get_pcb_user_save_td(struct thread *td) { Modified: head/sys/amd64/include/pcpu.h ============================================================================== --- head/sys/amd64/include/pcpu.h Thu Jul 19 15:36:36 2012 (r238622) +++ head/sys/amd64/include/pcpu.h Thu Jul 19 19:09:12 2012 (r238623) @@ -216,16 +216,29 @@ extern struct pcpu *pcpup; #define PCPU_PTR(member) __PCPU_PTR(pc_ ## member) #define PCPU_SET(member, val) __PCPU_SET(pc_ ## member, val) +#define OFFSETOF_CURTHREAD 0 static __inline __pure2 struct thread * __curthread(void) { struct thread *td; - __asm("movq %%gs:0,%0" : "=r" (td)); + __asm("movq %%gs:%1,%0" : "=r" (td) + : "m" (*(char *)OFFSETOF_CURTHREAD)); return (td); } #define curthread (__curthread()) +#define OFFSETOF_CURPCB 32 +static __inline __pure2 struct pcb * +__curpcb(void) +{ + struct pcb *pcb; + + __asm("movq %%gs:%1,%0" : "=r" (pcb) : "m" (*(char *)OFFSETOF_CURPCB)); + return (pcb); +} +#define curpcb (__curpcb()) + #define IS_BSP() (PCPU_GET(cpuid) == 0) #else /* !lint || defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE___TYPEOF) */