Date: Fri, 8 Nov 2019 12:06:03 +0000 (UTC) From: Andriy Gapon <avg@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org Subject: svn commit: r354534 - in stable/12/sys/cddl: compat/opensolaris/kern compat/opensolaris/sys contrib/opensolaris/common/atomic/aarch64 contrib/opensolaris/common/atomic/amd64 contrib/opensolaris/com... Message-ID: <201911081206.xA8C63OI035360@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: avg Date: Fri Nov 8 12:06:03 2019 New Revision: 354534 URL: https://svnweb.freebsd.org/changeset/base/354534 Log: MFC r353340,r353340: cleanup of illumos compatibility atomics Modified: stable/12/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c stable/12/sys/cddl/compat/opensolaris/sys/atomic.h stable/12/sys/cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S stable/12/sys/cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S stable/12/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S stable/12/sys/cddl/contrib/opensolaris/common/atomic/powerpc64/opensolaris_atomic.S stable/12/sys/cddl/contrib/opensolaris/common/atomic/sparc64/opensolaris_atomic.S Directory Properties: stable/12/ (props changed) Modified: stable/12/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c ============================================================================== --- stable/12/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c Fri Nov 8 11:09:50 2019 (r354533) +++ stable/12/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c Fri Nov 8 12:06:03 2019 (r354534) @@ -32,6 +32,9 @@ __FBSDID("$FreeBSD$"); #include <sys/mutex.h> #include <sys/atomic.h> +#if !defined(__LP64__) && !defined(__mips_n32) && \ + !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) + #ifdef _KERNEL #include <sys/kernel.h> @@ -52,8 +55,6 @@ atomic_init(void) } #endif -#if !defined(__LP64__) && !defined(__mips_n32) && \ - !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) void atomic_add_64(volatile uint64_t *target, int64_t delta) { @@ -94,7 +95,6 @@ atomic_load_64(volatile uint64_t *a) mtx_unlock(&atomic_mtx); return (ret); } -#endif uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta) @@ -107,27 +107,6 @@ atomic_add_64_nv(volatile uint64_t *target, int64_t de return (newval); } -#if defined(__powerpc__) || defined(__arm__) || defined(__mips__) -void -atomic_or_8(volatile uint8_t *target, uint8_t value) -{ - mtx_lock(&atomic_mtx); - *target |= value; - mtx_unlock(&atomic_mtx); -} -#endif - -uint8_t -atomic_or_8_nv(volatile uint8_t *target, uint8_t value) -{ - uint8_t newval; - - mtx_lock(&atomic_mtx); - newval = (*target |= value); - mtx_unlock(&atomic_mtx); - return (newval); -} - uint64_t atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval) { @@ -140,19 +119,7 @@ atomic_cas_64(volatile uint64_t *target, uint64_t cmp, mtx_unlock(&atomic_mtx); return (oldval); } - -uint32_t -atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval) -{ - uint32_t oldval; - - mtx_lock(&atomic_mtx); - oldval = *target; - if (oldval == cmp) - *target = newval; - mtx_unlock(&atomic_mtx); - return (oldval); -} +#endif void membar_producer(void) Modified: stable/12/sys/cddl/compat/opensolaris/sys/atomic.h ============================================================================== --- stable/12/sys/cddl/compat/opensolaris/sys/atomic.h Fri Nov 8 11:09:50 2019 (r354533) +++ stable/12/sys/cddl/compat/opensolaris/sys/atomic.h Fri Nov 8 12:06:03 2019 (r354534) @@ -32,42 +32,28 @@ #include <sys/types.h> #include <machine/atomic.h> -#define casptr(_a, _b, _c) \ - atomic_cmpset_ptr((volatile uintptr_t *)(_a), (uintptr_t)(_b), (uintptr_t) (_c)) -#define cas32 atomic_cmpset_32 - #if defined(__i386__) && (defined(_KERNEL) || defined(KLD_MODULE)) #define I386_HAVE_ATOMIC64 #endif +#if defined(__i386__) || defined(__amd64__) || defined(__arm__) +/* No spurious failures from fcmpset. */ +#define STRONG_FCMPSET +#endif + #if !defined(__LP64__) && !defined(__mips_n32) && \ !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) extern void atomic_add_64(volatile uint64_t *target, int64_t delta); extern void atomic_dec_64(volatile uint64_t *target); extern uint64_t atomic_swap_64(volatile uint64_t *a, uint64_t value); extern uint64_t atomic_load_64(volatile uint64_t *a); -#endif -#ifndef __sparc64__ -extern uint32_t atomic_cas_32(volatile uint32_t *target, uint32_t cmp, - uint32_t newval); +extern uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta); extern uint64_t atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval); #endif -extern uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta); -extern uint8_t atomic_or_8_nv(volatile uint8_t *target, uint8_t value); + extern void membar_producer(void); -#if defined(__sparc64__) || defined(__powerpc__) || defined(__arm__) || \ - defined(__mips__) || defined(__aarch64__) || defined(__riscv) -extern void atomic_or_8(volatile uint8_t *target, uint8_t value); -#else -static __inline void -atomic_or_8(volatile uint8_t *target, uint8_t value) -{ - atomic_set_8(target, value); -} -#endif - static __inline uint32_t atomic_add_32_nv(volatile uint32_t *target, int32_t delta) { @@ -81,6 +67,18 @@ atomic_add_int_nv(volatile u_int *target, int delta) } static __inline void +atomic_inc_32(volatile uint32_t *target) +{ + atomic_add_32(target, 1); +} + +static __inline uint32_t +atomic_inc_32_nv(volatile uint32_t *target) +{ + return (atomic_add_32_nv(target, 1)); +} + +static __inline void atomic_dec_32(volatile uint32_t *target) { atomic_subtract_32(target, 1); @@ -89,9 +87,27 @@ atomic_dec_32(volatile uint32_t *target) static __inline uint32_t atomic_dec_32_nv(volatile uint32_t *target) { - return (atomic_fetchadd_32(target, -1) - 1); + return (atomic_add_32_nv(target, -1)); } +#ifndef __sparc64__ +static inline uint32_t +atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval) +{ +#ifdef STRONG_FCMPSET + (void)atomic_fcmpset_32(target, &cmp, newval); +#else + uint32_t expected = cmp; + + do { + if (atomic_fcmpset_32(target, &cmp, newval)) + break; + } while (cmp == expected); +#endif + return (cmp); +} +#endif + #if defined(__LP64__) || defined(__mips_n32) || \ defined(ARM_HAVE_ATOMIC64) || defined(I386_HAVE_ATOMIC64) static __inline void @@ -99,19 +115,31 @@ atomic_dec_64(volatile uint64_t *target) { atomic_subtract_64(target, 1); } -#endif -static __inline void -atomic_inc_32(volatile uint32_t *target) +static inline uint64_t +atomic_add_64_nv(volatile uint64_t *target, int64_t delta) { - atomic_add_32(target, 1); + return (atomic_fetchadd_64(target, delta) + delta); } -static __inline uint32_t -atomic_inc_32_nv(volatile uint32_t *target) +#ifndef __sparc64__ +static inline uint64_t +atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval) { - return (atomic_add_32_nv(target, 1)); +#ifdef STRONG_FCMPSET + (void)atomic_fcmpset_64(target, &cmp, newval); +#else + uint64_t expected = cmp; + + do { + if (atomic_fcmpset_64(target, &cmp, newval)) + break; + } while (cmp == expected); +#endif + return (cmp); } +#endif +#endif static __inline void atomic_inc_64(volatile uint64_t *target) Modified: stable/12/sys/cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S ============================================================================== --- stable/12/sys/cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S Fri Nov 8 11:09:50 2019 (r354533) +++ stable/12/sys/cddl/contrib/opensolaris/common/atomic/aarch64/opensolaris_atomic.S Fri Nov 8 12:06:03 2019 (r354534) @@ -28,58 +28,6 @@ #include <machine/asm.h> -/* - * uint64_t atomic_add_64_nv(volatile uint64_t *target, int64_t delta) - */ -ENTRY(atomic_add_64_nv) -1: ldxr x2, [x0] /* Load *target */ - add x2, x2, x1 /* x2 = x2 + delta */ - stxr w3, x2, [x0] /* Store *target */ - cbnz w3, 1b /* Check if the store succeeded */ - mov x0, x2 /* Return the new value */ - ret -END(atomic_add_64_nv) - -/* - * uint32_t - * atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval) - */ -ENTRY(atomic_cas_32) -1: ldxr w3, [x0] /* Load *target */ - cmp w3, w1 /* Does *targe == cmp? */ - b.ne 2f /* If not exit */ - stxr w4, w2, [x0] /* Store newval to *target */ - cbnz w4, 1b /* Check if the store succeeded */ -2: mov w0, w3 /* Return the old value */ - ret -END(atomic_cas_32) - -/* - * uint64_t - * atomic_cas_64(volatile uint64_t *target, uint64_t cmp, uint64_t newval) - */ -ENTRY(atomic_cas_64) -1: ldxr x3, [x0] /* Load *target */ - cmp x3, x1 /* Does *targe == cmp? */ - b.ne 2f /* If not exit */ - stxr w4, x2, [x0] /* Store newval to *target */ - cbnz w4, 1b /* Check if the store succeeded */ -2: mov x0, x3 /* Return the old value */ - ret -END(atomic_cas_64) - -/* - * uint8_t atomic_or_8_nv(volatile uint8_t *target, uint8_t value) - */ -ENTRY(atomic_or_8_nv) -1: ldxrb w2, [x0] /* Load *target */ - orr w2, w2, w1 /* x2 = x2 | delta */ - stxrb w3, w2, [x0] /* Store *target */ - cbnz w3, 1b /* Check if the store succeeded */ - mov w0, w2 /* Return the new value */ - ret -END(atomic_or_8_nv) - ENTRY(membar_producer) dmb ish ret Modified: stable/12/sys/cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S ============================================================================== --- stable/12/sys/cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S Fri Nov 8 11:09:50 2019 (r354533) +++ stable/12/sys/cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S Fri Nov 8 12:06:03 2019 (r354534) @@ -28,40 +28,6 @@ #define _ASM #include <sys/asm_linkage.h> - ENTRY(atomic_add_64_nv) - mov %rsi, %rax // %rax = delta addend - lock - xaddq %rsi, (%rdi) // %rsi = old value, (%rdi) = sum - addq %rsi, %rax // new value = original value + delta - ret - SET_SIZE(atomic_add_64_nv) - - ENTRY(atomic_or_8_nv) - movb (%rdi), %al // %al = old value -1: - movb %sil, %cl - orb %al, %cl // %cl = new value - lock - cmpxchgb %cl, (%rdi) // try to stick it in - jne 1b - movzbl %cl, %eax // return new value - ret - SET_SIZE(atomic_or_8_nv) - - ENTRY(atomic_cas_32) - movl %esi, %eax - lock - cmpxchgl %edx, (%rdi) - ret - SET_SIZE(atomic_cas_32) - - ENTRY(atomic_cas_64) - movq %rsi, %rax - lock - cmpxchgq %rdx, (%rdi) - ret - SET_SIZE(atomic_cas_64) - ENTRY(membar_producer) sfence ret Modified: stable/12/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S ============================================================================== --- stable/12/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S Fri Nov 8 11:09:50 2019 (r354533) +++ stable/12/sys/cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S Fri Nov 8 12:06:03 2019 (r354534) @@ -89,28 +89,6 @@ SET_SIZE(atomic_add_64_nv) SET_SIZE(atomic_add_64) - ENTRY(atomic_or_8_nv) - movl 4(%esp), %edx // %edx = target address - movb (%edx), %al // %al = old value -1: - movl 8(%esp), %ecx // %ecx = delta - orb %al, %cl // %cl = new value - lock - cmpxchgb %cl, (%edx) // try to stick it in - jne 1b - movzbl %cl, %eax // return new value - ret - SET_SIZE(atomic_or_8_nv) - - ENTRY(atomic_cas_32) - movl 4(%esp), %edx - movl 8(%esp), %eax - movl 12(%esp), %ecx - lock - cmpxchgl %ecx, (%edx) - ret - SET_SIZE(atomic_cas_32) - ENTRY(atomic_cas_64) pushl %ebx pushl %esi Modified: stable/12/sys/cddl/contrib/opensolaris/common/atomic/powerpc64/opensolaris_atomic.S ============================================================================== --- stable/12/sys/cddl/contrib/opensolaris/common/atomic/powerpc64/opensolaris_atomic.S Fri Nov 8 11:09:50 2019 (r354533) +++ stable/12/sys/cddl/contrib/opensolaris/common/atomic/powerpc64/opensolaris_atomic.S Fri Nov 8 12:06:03 2019 (r354534) @@ -27,61 +27,6 @@ #include <machine/asm.h> -ENTRY(atomic_add_64_nv) - 1: ldarx %r5,0,%r3 - add %r5,%r4,%r5 - stdcx. %r5,0,%r3 - bne- 1b - - mr %r3,%r5 - blr - -ENTRY(atomic_cas_32) - 1: lwarx %r6,0,%r3 - cmplw %r6,%r4 - bne 2f - stwcx. %r5,0,%r3 - bne- 1b - b 3f - - 2: stwcx. %r6,0,%r3 /* clear reservation */ - - 3: mr %r3,%r6 - blr - -ENTRY(atomic_cas_64) - 1: ldarx %r6,0,%r3 - cmpld %r6,%r4 - bne 2f - stdcx. %r5,0,%r3 - bne- 1b - b 3f - - 2: stdcx. %r6,0,%r3 /* clear reservation */ - - 3: mr %r3,%r6 - blr - -ENTRY(atomic_or_8_nv) - li %r6,3 - andc. %r6,%r3,%r6 /* r6 = r3 & ~4 */ - addi %r7,%r6,3 - sub %r7,%r7,%r3 /* offset in r7 */ - sldi %r7,%r7,3 /* bits to shift in r7 */ - - rlwinm %r4,%r4,0,24,31 /* mask and rotate the argument */ - slw %r4,%r4,%r7 - - 1: lwarx %r5,0,%r6 - or %r5,%r4,%r5 - stwcx. %r5,0,%r6 - bne- 1b - - srw %r3,%r5,%r7 - rlwinm %r3,%r3,0,24,31 /* mask return value */ - - blr - ENTRY(membar_producer) eieio blr Modified: stable/12/sys/cddl/contrib/opensolaris/common/atomic/sparc64/opensolaris_atomic.S ============================================================================== --- stable/12/sys/cddl/contrib/opensolaris/common/atomic/sparc64/opensolaris_atomic.S Fri Nov 8 11:09:50 2019 (r354533) +++ stable/12/sys/cddl/contrib/opensolaris/common/atomic/sparc64/opensolaris_atomic.S Fri Nov 8 12:06:03 2019 (r354534) @@ -40,67 +40,6 @@ #endif /* - * NOTE: If atomic_add_64 and atomic_add_64_nv are ever - * separated, you need to also edit the libc sparcv9 platform - * specific mapfile and remove the NODYNSORT attribute - * from atomic_add_64_nv. - */ - ENTRY(atomic_add_64) - ALTENTRY(atomic_add_64_nv) - ALTENTRY(atomic_add_ptr) - ALTENTRY(atomic_add_ptr_nv) - ALTENTRY(atomic_add_long) - ALTENTRY(atomic_add_long_nv) -add_64: - ldx [%o0], %o2 -1: - add %o2, %o1, %o3 - casxa [%o0] __ASI_ATOMIC, %o2, %o3 - cmp %o2, %o3 - bne,a,pn %xcc, 1b - mov %o3, %o2 - retl - add %o2, %o1, %o0 ! return new value - SET_SIZE(atomic_add_long_nv) - SET_SIZE(atomic_add_long) - SET_SIZE(atomic_add_ptr_nv) - SET_SIZE(atomic_add_ptr) - SET_SIZE(atomic_add_64_nv) - SET_SIZE(atomic_add_64) - - /* - * NOTE: If atomic_or_8 and atomic_or_8_nv are ever - * separated, you need to also edit the libc sparcv9 platform - * specific mapfile and remove the NODYNSORT attribute - * from atomic_or_8_nv. - */ - ENTRY(atomic_or_8) - ALTENTRY(atomic_or_8_nv) - ALTENTRY(atomic_or_uchar) - and %o0, 0x3, %o4 ! %o4 = byte offset, left-to-right - xor %o4, 0x3, %g1 ! %g1 = byte offset, right-to-left - sll %g1, 3, %g1 ! %g1 = bit offset, right-to-left - set 0xff, %o3 ! %o3 = mask - sll %o3, %g1, %o3 ! %o3 = shifted to bit offset - sll %o1, %g1, %o1 ! %o1 = shifted to bit offset - and %o1, %o3, %o1 ! %o1 = single byte value - andn %o0, 0x3, %o0 ! %o0 = word address - ld [%o0], %o2 ! read old value -1: - or %o2, %o1, %o5 ! or in the new value - casa [%o0] __ASI_ATOMIC, %o2, %o5 - cmp %o2, %o5 - bne,a,pn %icc, 1b - mov %o5, %o2 ! %o2 = old value - or %o2, %o1, %o5 - and %o5, %o3, %o5 - retl - srl %o5, %g1, %o0 ! %o0 = new value - SET_SIZE(atomic_or_uchar) - SET_SIZE(atomic_or_8_nv) - SET_SIZE(atomic_or_8) - - /* * Spitfires and Blackbirds have a problem with membars in the * delay slot (SF_ERRATA_51). For safety's sake, we assume * that the whole world needs the workaround.
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201911081206.xA8C63OI035360>