Date: Fri, 14 Oct 2016 15:53:48 +0000 (UTC) From: Andrew Turner <andrew@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r307321 - in head/sys/arm64: arm64 include Message-ID: <201610141553.u9EFrmT7062155@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: andrew Date: Fri Oct 14 15:53:48 2016 New Revision: 307321 URL: https://svnweb.freebsd.org/changeset/base/307321 Log: Rework how we store the VFP registers in the pcb. This will be used when creating a floating-point context within the kernel without having to move the stored values in memory. Sponsored by: The FreeBSD Foundation Modified: head/sys/arm64/arm64/machdep.c head/sys/arm64/arm64/vfp.c head/sys/arm64/arm64/vm_machdep.c head/sys/arm64/include/pcb.h head/sys/arm64/include/vfp.h Modified: head/sys/arm64/arm64/machdep.c ============================================================================== --- head/sys/arm64/arm64/machdep.c Fri Oct 14 15:16:44 2016 (r307320) +++ head/sys/arm64/arm64/machdep.c Fri Oct 14 15:53:48 2016 (r307321) @@ -192,9 +192,12 @@ fill_fpregs(struct thread *td, struct fp */ vfp_save_state(td, pcb); - memcpy(regs->fp_q, pcb->pcb_vfp, sizeof(regs->fp_q)); - regs->fp_cr = pcb->pcb_fpcr; - regs->fp_sr = pcb->pcb_fpsr; + KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate, + ("Called fill_fpregs while the kernel is using the VFP")); + memcpy(regs->fp_q, pcb->pcb_fpustate.vfp_regs, + sizeof(regs->fp_q)); + regs->fp_cr = pcb->pcb_fpustate.vfp_fpcr; + regs->fp_sr = pcb->pcb_fpustate.vfp_fpsr; } else #endif memset(regs->fp_q, 0, sizeof(regs->fp_q)); @@ -208,9 +211,11 @@ set_fpregs(struct thread *td, struct fpr struct pcb *pcb; pcb = td->td_pcb; - memcpy(pcb->pcb_vfp, regs->fp_q, sizeof(regs->fp_q)); - pcb->pcb_fpcr = regs->fp_cr; - pcb->pcb_fpsr = regs->fp_sr; + KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate, + ("Called set_fpregs while the kernel is using the VFP")); + memcpy(pcb->pcb_fpustate.vfp_regs, regs->fp_q, sizeof(regs->fp_q)); + pcb->pcb_fpustate.vfp_fpcr = regs->fp_cr; + pcb->pcb_fpustate.vfp_fpsr = regs->fp_sr; #endif return (0); } @@ -334,10 +339,12 @@ get_fpcontext(struct thread *td, mcontex */ vfp_save_state(td, curpcb); - memcpy(mcp->mc_fpregs.fp_q, curpcb->pcb_vfp, + KASSERT(curpcb->pcb_fpusaved == &curpcb->pcb_fpustate, + ("Called get_fpcontext while the kernel is using the VFP")); + memcpy(mcp->mc_fpregs.fp_q, curpcb->pcb_fpustate.vfp_regs, sizeof(mcp->mc_fpregs)); - mcp->mc_fpregs.fp_cr = curpcb->pcb_fpcr; - mcp->mc_fpregs.fp_sr = curpcb->pcb_fpsr; + mcp->mc_fpregs.fp_cr = curpcb->pcb_fpustate.vfp_fpcr; + mcp->mc_fpregs.fp_sr = curpcb->pcb_fpustate.vfp_fpsr; mcp->mc_fpregs.fp_flags = curpcb->pcb_fpflags; mcp->mc_flags |= _MC_FP_VALID; } @@ -363,10 +370,12 @@ set_fpcontext(struct thread *td, mcontex */ vfp_discard(td); - memcpy(curpcb->pcb_vfp, mcp->mc_fpregs.fp_q, + KASSERT(curpcb->pcb_fpusaved == &curpcb->pcb_fpustate, + ("Called set_fpcontext while the kernel is using the VFP")); + memcpy(curpcb->pcb_fpustate.vfp_regs, mcp->mc_fpregs.fp_q, sizeof(mcp->mc_fpregs)); - curpcb->pcb_fpcr = mcp->mc_fpregs.fp_cr; - curpcb->pcb_fpsr = mcp->mc_fpregs.fp_sr; + curpcb->pcb_fpustate.vfp_fpcr = mcp->mc_fpregs.fp_cr; + curpcb->pcb_fpustate.vfp_fpsr = mcp->mc_fpregs.fp_sr; curpcb->pcb_fpflags = mcp->mc_fpregs.fp_flags; } @@ -607,6 +616,7 @@ init_proc0(vm_offset_t kstack) thread0.td_kstack = kstack; thread0.td_pcb = (struct pcb *)(thread0.td_kstack) - 1; thread0.td_pcb->pcb_fpflags = 0; + thread0.td_pcb->pcb_fpusaved = &thread0.td_pcb->pcb_fpustate; thread0.td_pcb->pcb_vfpcpu = UINT_MAX; thread0.td_frame = &proc0_tf; pcpup->pc_curpcb = thread0.td_pcb; Modified: head/sys/arm64/arm64/vfp.c ============================================================================== --- head/sys/arm64/arm64/vfp.c Fri Oct 14 15:16:44 2016 (r307320) +++ head/sys/arm64/arm64/vfp.c Fri Oct 14 15:53:48 2016 (r307321) @@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/systm.h> #include <sys/kernel.h> +#include <sys/malloc.h> #include <sys/pcpu.h> #include <sys/proc.h> @@ -42,7 +43,10 @@ __FBSDID("$FreeBSD$"); #include <machine/vfp.h> /* Sanity check we can store all the VFP registers */ -CTASSERT(sizeof(((struct pcb *)0)->pcb_vfp) == 16 * 32); +CTASSERT(sizeof(((struct pcb *)0)->pcb_fpustate.vfp_regs) == 16 * 32); + +static MALLOC_DEFINE(M_FPUKERN_CTX, "fpukern_ctx", + "Kernel contexts for VFP state"); static void vfp_enable(void) @@ -81,11 +85,73 @@ vfp_discard(struct thread *td) vfp_disable(); } -void -vfp_save_state(struct thread *td, struct pcb *pcb) +static void +vfp_store(struct vfpstate *state) +{ + __int128_t *vfp_state; + uint64_t fpcr, fpsr; + + vfp_state = state->vfp_regs; + __asm __volatile( + "mrs %0, fpcr \n" + "mrs %1, fpsr \n" + "stp q0, q1, [%2, #16 * 0]\n" + "stp q2, q3, [%2, #16 * 2]\n" + "stp q4, q5, [%2, #16 * 4]\n" + "stp q6, q7, [%2, #16 * 6]\n" + "stp q8, q9, [%2, #16 * 8]\n" + "stp q10, q11, [%2, #16 * 10]\n" + "stp q12, q13, [%2, #16 * 12]\n" + "stp q14, q15, [%2, #16 * 14]\n" + "stp q16, q17, [%2, #16 * 16]\n" + "stp q18, q19, [%2, #16 * 18]\n" + "stp q20, q21, [%2, #16 * 20]\n" + "stp q22, q23, [%2, #16 * 22]\n" + "stp q24, q25, [%2, #16 * 24]\n" + "stp q26, q27, [%2, #16 * 26]\n" + "stp q28, q29, [%2, #16 * 28]\n" + "stp q30, q31, [%2, #16 * 30]\n" + : "=&r"(fpcr), "=&r"(fpsr) : "r"(vfp_state)); + + state->vfp_fpcr = fpcr; + state->vfp_fpsr = fpsr; +} + +static void +vfp_restore(struct vfpstate *state) { __int128_t *vfp_state; uint64_t fpcr, fpsr; + + vfp_state = state->vfp_regs; + fpcr = state->vfp_fpcr; + fpsr = state->vfp_fpsr; + + __asm __volatile( + "ldp q0, q1, [%2, #16 * 0]\n" + "ldp q2, q3, [%2, #16 * 2]\n" + "ldp q4, q5, [%2, #16 * 4]\n" + "ldp q6, q7, [%2, #16 * 6]\n" + "ldp q8, q9, [%2, #16 * 8]\n" + "ldp q10, q11, [%2, #16 * 10]\n" + "ldp q12, q13, [%2, #16 * 12]\n" + "ldp q14, q15, [%2, #16 * 14]\n" + "ldp q16, q17, [%2, #16 * 16]\n" + "ldp q18, q19, [%2, #16 * 18]\n" + "ldp q20, q21, [%2, #16 * 20]\n" + "ldp q22, q23, [%2, #16 * 22]\n" + "ldp q24, q25, [%2, #16 * 24]\n" + "ldp q26, q27, [%2, #16 * 26]\n" + "ldp q28, q29, [%2, #16 * 28]\n" + "ldp q30, q31, [%2, #16 * 30]\n" + "msr fpcr, %0 \n" + "msr fpsr, %1 \n" + : : "r"(fpcr), "r"(fpsr), "r"(vfp_state)); +} + +void +vfp_save_state(struct thread *td, struct pcb *pcb) +{ uint32_t cpacr; KASSERT(pcb != NULL, ("NULL vfp pcb")); @@ -104,31 +170,7 @@ vfp_save_state(struct thread *td, struct KASSERT(PCPU_GET(fpcurthread) == td, ("Storing an invalid VFP state")); - vfp_state = pcb->pcb_vfp; - __asm __volatile( - "mrs %0, fpcr \n" - "mrs %1, fpsr \n" - "stp q0, q1, [%2, #16 * 0]\n" - "stp q2, q3, [%2, #16 * 2]\n" - "stp q4, q5, [%2, #16 * 4]\n" - "stp q6, q7, [%2, #16 * 6]\n" - "stp q8, q9, [%2, #16 * 8]\n" - "stp q10, q11, [%2, #16 * 10]\n" - "stp q12, q13, [%2, #16 * 12]\n" - "stp q14, q15, [%2, #16 * 14]\n" - "stp q16, q17, [%2, #16 * 16]\n" - "stp q18, q19, [%2, #16 * 18]\n" - "stp q20, q21, [%2, #16 * 20]\n" - "stp q22, q23, [%2, #16 * 22]\n" - "stp q24, q25, [%2, #16 * 24]\n" - "stp q26, q27, [%2, #16 * 26]\n" - "stp q28, q29, [%2, #16 * 28]\n" - "stp q30, q31, [%2, #16 * 30]\n" - : "=&r"(fpcr), "=&r"(fpsr) : "r"(vfp_state)); - - pcb->pcb_fpcr = fpcr; - pcb->pcb_fpsr = fpsr; - + vfp_store(pcb->pcb_fpusaved); dsb(ish); vfp_disable(); } @@ -138,8 +180,6 @@ vfp_save_state(struct thread *td, struct void vfp_restore_state(void) { - __int128_t *vfp_state; - uint64_t fpcr, fpsr; struct pcb *curpcb; u_int cpu; @@ -158,31 +198,7 @@ vfp_restore_state(void) */ if (PCPU_GET(fpcurthread) != curthread || cpu != curpcb->pcb_vfpcpu) { - vfp_state = curthread->td_pcb->pcb_vfp; - fpcr = curthread->td_pcb->pcb_fpcr; - fpsr = curthread->td_pcb->pcb_fpsr; - - __asm __volatile( - "ldp q0, q1, [%2, #16 * 0]\n" - "ldp q2, q3, [%2, #16 * 2]\n" - "ldp q4, q5, [%2, #16 * 4]\n" - "ldp q6, q7, [%2, #16 * 6]\n" - "ldp q8, q9, [%2, #16 * 8]\n" - "ldp q10, q11, [%2, #16 * 10]\n" - "ldp q12, q13, [%2, #16 * 12]\n" - "ldp q14, q15, [%2, #16 * 14]\n" - "ldp q16, q17, [%2, #16 * 16]\n" - "ldp q18, q19, [%2, #16 * 18]\n" - "ldp q20, q21, [%2, #16 * 20]\n" - "ldp q22, q23, [%2, #16 * 22]\n" - "ldp q24, q25, [%2, #16 * 24]\n" - "ldp q26, q27, [%2, #16 * 26]\n" - "ldp q28, q29, [%2, #16 * 28]\n" - "ldp q30, q31, [%2, #16 * 30]\n" - "msr fpcr, %0 \n" - "msr fpsr, %1 \n" - : : "r"(fpcr), "r"(fpsr), "r"(vfp_state)); - + vfp_restore(curthread->td_pcb->pcb_fpusaved); PCPU_SET(fpcurthread, curthread); curpcb->pcb_vfpcpu = cpu; } Modified: head/sys/arm64/arm64/vm_machdep.c ============================================================================== --- head/sys/arm64/arm64/vm_machdep.c Fri Oct 14 15:16:44 2016 (r307320) +++ head/sys/arm64/arm64/vm_machdep.c Fri Oct 14 15:53:48 2016 (r307321) @@ -101,6 +101,7 @@ cpu_fork(struct thread *td1, struct proc td2->td_pcb->pcb_x[9] = (uintptr_t)td2; td2->td_pcb->pcb_x[PCB_LR] = (uintptr_t)fork_trampoline; td2->td_pcb->pcb_sp = (uintptr_t)td2->td_frame; + td2->td_pcb->pcb_fpusaved = &td2->td_pcb->pcb_fpustate; td2->td_pcb->pcb_vfpcpu = UINT_MAX; /* Setup to release spin count in fork_exit(). */ @@ -169,6 +170,7 @@ cpu_copy_thread(struct thread *td, struc td->td_pcb->pcb_x[9] = (uintptr_t)td; td->td_pcb->pcb_x[PCB_LR] = (uintptr_t)fork_trampoline; td->td_pcb->pcb_sp = (uintptr_t)td->td_frame; + td->td_pcb->pcb_fpusaved = &td->td_pcb->pcb_fpustate; td->td_pcb->pcb_vfpcpu = UINT_MAX; /* Setup to release spin count in fork_exit(). */ @@ -246,6 +248,7 @@ cpu_fork_kthread_handler(struct thread * td->td_pcb->pcb_x[9] = (uintptr_t)arg; td->td_pcb->pcb_x[PCB_LR] = (uintptr_t)fork_trampoline; td->td_pcb->pcb_sp = (uintptr_t)td->td_frame; + td->td_pcb->pcb_fpusaved = &td->td_pcb->pcb_fpustate; td->td_pcb->pcb_vfpcpu = UINT_MAX; } Modified: head/sys/arm64/include/pcb.h ============================================================================== --- head/sys/arm64/include/pcb.h Fri Oct 14 15:16:44 2016 (r307320) +++ head/sys/arm64/include/pcb.h Fri Oct 14 15:53:48 2016 (r307321) @@ -31,6 +31,8 @@ #ifndef LOCORE +#include <machine/vfp.h> + struct trapframe; #define PCB_LR 30 @@ -49,13 +51,17 @@ struct pcb { #define PCB_SINGLE_STEP_SHIFT 0 #define PCB_SINGLE_STEP (1 << PCB_SINGLE_STEP_SHIFT) - /* Place last to simplify the asm to access the rest if the struct */ - __uint128_t pcb_vfp[32]; - uint32_t pcb_fpcr; - uint32_t pcb_fpsr; + struct vfpstate *pcb_fpusaved; int pcb_fpflags; #define PCB_FP_STARTED 0x01 u_int pcb_vfpcpu; /* Last cpu this thread ran VFP code */ + + /* + * The userspace VFP state. The pcb_fpusaved pointer will point to + * this unless the kernel has allocated a VFP context. + * Place last to simplify the asm to access the rest if the struct. + */ + struct vfpstate pcb_fpustate; }; #ifdef _KERNEL Modified: head/sys/arm64/include/vfp.h ============================================================================== --- head/sys/arm64/include/vfp.h Fri Oct 14 15:16:44 2016 (r307320) +++ head/sys/arm64/include/vfp.h Fri Oct 14 15:53:48 2016 (r307321) @@ -35,6 +35,12 @@ #ifdef _KERNEL #ifndef LOCORE +struct vfpstate { + __uint128_t vfp_regs[32]; + uint32_t vfp_fpcr; + uint32_t vfp_fpsr; +}; + void vfp_init(void); void vfp_discard(struct thread *); void vfp_restore_state(void);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201610141553.u9EFrmT7062155>