Date: Wed, 6 Apr 2011 23:59:59 +0000 (UTC) From: Jung-uk Kim <jkim@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r220404 - in head/sys: i386/i386 i386/include pc98/pc98 Message-ID: <201104062359.p36NxxxT022808@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jkim Date: Wed Apr 6 23:59:59 2011 New Revision: 220404 URL: http://svn.freebsd.org/changeset/base/220404 Log: Implement atomic_load_acq_64(9) and atomic_store_rel_64(9) for i386. These functions are implemented with CMPXCHG8B instruction where it is available, i. e., all Pentium-class and later processors. Note this instruction is also used for atomic_store_rel_64() because a simple XCHG-like instruction for 64-bit memory access does not exist, unfortunately. If the processor lacks the instruction, i. e., 80486-class CPUs, two 32-bit load/store are performed with interrupt temporarily disabled, assuming it does not support SMP. Although this assumption may be little naive, it is true in reality. This implementation is inspired by Linux. Modified: head/sys/i386/i386/machdep.c head/sys/i386/include/atomic.h head/sys/pc98/pc98/machdep.c Modified: head/sys/i386/i386/machdep.c ============================================================================== --- head/sys/i386/i386/machdep.c Wed Apr 6 20:54:26 2011 (r220403) +++ head/sys/i386/i386/machdep.c Wed Apr 6 23:59:59 2011 (r220404) @@ -1497,6 +1497,22 @@ idle_sysctl(SYSCTL_HANDLER_ARGS) SYSCTL_PROC(_machdep, OID_AUTO, idle, CTLTYPE_STRING | CTLFLAG_RW, 0, 0, idle_sysctl, "A", "currently selected idle function"); +uint64_t (*atomic_load_acq_64)(volatile uint64_t *) = + atomic_load_acq_64_i386; +void (*atomic_store_rel_64)(volatile uint64_t *, uint64_t) = + atomic_store_rel_64_i386; + +static void +cpu_probe_cmpxchg8b(void) +{ + + if ((cpu_feature & CPUID_CX8) != 0 || + cpu_vendor_id == CPU_VENDOR_RISE) { + atomic_load_acq_64 = atomic_load_acq_64_i586; + atomic_store_rel_64 = atomic_store_rel_64_i586; + } +} + /* * Reset registers to default values on exec. */ @@ -2730,6 +2746,7 @@ init386(first) thread0.td_pcb->pcb_gsd = PCPU_GET(fsgs_gdt)[1]; cpu_probe_amdc1e(); + cpu_probe_cmpxchg8b(); } #else @@ -3006,6 +3023,7 @@ init386(first) thread0.td_frame = &proc0_tf; cpu_probe_amdc1e(); + cpu_probe_cmpxchg8b(); } #endif Modified: head/sys/i386/include/atomic.h ============================================================================== --- head/sys/i386/include/atomic.h Wed Apr 6 20:54:26 2011 (r220403) +++ head/sys/i386/include/atomic.h Wed Apr 6 23:59:59 2011 (r220404) @@ -120,6 +120,87 @@ atomic_##NAME##_barr_##TYPE(volatile u_# } \ struct __hack +#if defined(_KERNEL) && !defined(WANT_FUNCTIONS) + +/* I486 does not support SMP or CMPXCHG8B. */ +static __inline uint64_t +atomic_load_acq_64_i386(volatile uint64_t *p) +{ + volatile uint32_t *high, *low; + uint64_t res; + + low = (volatile uint32_t *)p; + high = (volatile uint32_t *)p + 1; + __asm __volatile( + " pushfl ; " + " cli ; " + " movl %1,%%eax ; " + " movl %2,%%edx ; " + " popfl" + : "=&A" (res) /* 0 */ + : "m" (*low), /* 1 */ + "m" (*high) /* 2 */ + : "memory"); + + return (res); +} + +static __inline void +atomic_store_rel_64_i386(volatile uint64_t *p, uint64_t v) +{ + volatile uint32_t *high, *low; + + low = (volatile uint32_t *)p; + high = (volatile uint32_t *)p + 1; + __asm __volatile( + " pushfl ; " + " cli ; " + " movl %%eax,%0 ; " + " movl %%edx,%1 ; " + " popfl" + : "=m" (*low), /* 0 */ + "=m" (*high) /* 1 */ + : "A" (v) /* 2 */ + : "memory"); +} + +static __inline uint64_t +atomic_load_acq_64_i586(volatile uint64_t *p) +{ + uint64_t res; + + __asm __volatile( + " movl %%ebx,%%eax ; " + " movl %%ecx,%%edx ; " + " " MPLOCKED " " + " cmpxchg8b %2" + : "=&A" (res), /* 0 */ + "=m" (*p) /* 1 */ + : "m" (*p) /* 2 */ + : "memory", "cc"); + + return (res); +} + +static __inline void +atomic_store_rel_64_i586(volatile uint64_t *p, uint64_t v) +{ + + __asm __volatile( + " movl %%eax,%%ebx ; " + " movl %%edx,%%ecx ; " + "1: " + " " MPLOCKED " " + " cmpxchg8b %2 ; " + " jne 1b" + : "=m" (*p), /* 0 */ + "+A" (v) /* 1 */ + : "m" (*p) /* 2 */ + : "ebx", "ecx", "memory", "cc"); +} + +#endif /* _KERNEL && !WANT_FUNCTIONS */ + /* * Atomic compare and set, used by the mutex functions * @@ -292,6 +373,11 @@ ATOMIC_STORE_LOAD(long, "cmpxchgl %0,%1" #ifndef WANT_FUNCTIONS +#ifdef _KERNEL +extern uint64_t (*atomic_load_acq_64)(volatile uint64_t *); +extern void (*atomic_store_rel_64)(volatile uint64_t *, uint64_t); +#endif + static __inline int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src) { Modified: head/sys/pc98/pc98/machdep.c ============================================================================== --- head/sys/pc98/pc98/machdep.c Wed Apr 6 20:54:26 2011 (r220403) +++ head/sys/pc98/pc98/machdep.c Wed Apr 6 23:59:59 2011 (r220404) @@ -1330,6 +1330,21 @@ idle_sysctl(SYSCTL_HANDLER_ARGS) SYSCTL_PROC(_machdep, OID_AUTO, idle, CTLTYPE_STRING | CTLFLAG_RW, 0, 0, idle_sysctl, "A", "currently selected idle function"); +uint64_t (*atomic_load_acq_64)(volatile uint64_t *) = + atomic_load_acq_64_i386; +void (*atomic_store_rel_64)(volatile uint64_t *, uint64_t) = + atomic_store_rel_64_i386; + +static void +cpu_probe_cmpxchg8b(void) +{ + + if ((cpu_feature & CPUID_CX8) != 0) { + atomic_load_acq_64 = atomic_load_acq_64_i586; + atomic_store_rel_64 = atomic_store_rel_64_i586; + } +} + /* * Reset registers to default values on exec. */ @@ -2344,6 +2359,8 @@ init386(first) thread0.td_pcb->pcb_cr3 = (int)IdlePTD; thread0.td_pcb->pcb_ext = 0; thread0.td_frame = &proc0_tf; + + cpu_probe_cmpxchg8b(); } void
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201104062359.p36NxxxT022808>