Date: Wed, 25 Mar 2020 23:12:43 +0000 (UTC) From: Conrad Meyer <cem@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r359311 - in head/sys: amd64/include arm/include arm64/include i386/include kern powerpc/include sys Message-ID: <202003252312.02PNChC7051461@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: cem Date: Wed Mar 25 23:12:43 2020 New Revision: 359311 URL: https://svnweb.freebsd.org/changeset/base/359311 Log: Expand generic subword atomic primitives The goal of this change is to make the atomic_load_acq_{8,16}, atomic_testandset{,_acq}_long, and atomic_testandclear_long primitives available in MI-namespace. The second goal is to get this draft out of my local tree, as anything that requires a full tinderbox is a big burden out of tree. MD specifics can be refined individually afterwards. The generic implementations may not be ideal for your architecture; feel free to implement better versions. If no subword_atomic definitions are needed, the include can be removed from your arch's machine/atomic.h. Generic definitions are guarded by defined macros of the same name. To avoid picking up conflicting generic definitions, some macro defines are added to various MD machine/atomic.h to register an existing implementation. Include _atomic_subword.h in arm and arm64 machine/atomic.h. For some odd reason, KCSAN only generates some versions of primitives. Generate the _acq variants of atomic_load.*_8, atomic_load.*_16, and atomic_testandset.*_long. There are other questionably disabled primitives, but I didn't run into them, so I left them alone. KCSAN is only built for amd64 in tinderbox for now. Add atomic_subword implementations of atomic_load_acq_{8,16} implemented using masking and atomic_load_acq_32. Add generic atomic_subword implementations of atomic_testandset_long(), atomic_testandclear_long(), and atomic_testandset_acq_long(), using atomic_fcmpset_long() and atomic_fcmpset_acq_long(). On x86, add atomic_testandset_acq_long as an alias for atomic_testandset_long. Reviewed by: kevans, rlibby (previous versions both) Differential Revision: https://reviews.freebsd.org/D22963 Modified: head/sys/amd64/include/atomic.h head/sys/arm/include/atomic-v4.h head/sys/arm/include/atomic-v6.h head/sys/arm/include/atomic.h head/sys/arm64/include/atomic.h head/sys/i386/include/atomic.h head/sys/kern/subr_csan.c head/sys/powerpc/include/atomic.h head/sys/sys/_atomic_subword.h head/sys/sys/_cscan_atomic.h Modified: head/sys/amd64/include/atomic.h ============================================================================== --- head/sys/amd64/include/atomic.h Wed Mar 25 23:06:04 2020 (r359310) +++ head/sys/amd64/include/atomic.h Wed Mar 25 23:12:43 2020 (r359311) @@ -553,6 +553,7 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_readandclear_int(p) atomic_swap_int(p, 0) #define atomic_readandclear_long(p) atomic_swap_long(p, 0) +#define atomic_testandset_acq_long atomic_testandset_long /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char Modified: head/sys/arm/include/atomic-v4.h ============================================================================== --- head/sys/arm/include/atomic-v4.h Wed Mar 25 23:06:04 2020 (r359310) +++ head/sys/arm/include/atomic-v4.h Wed Mar 25 23:12:43 2020 (r359311) @@ -521,8 +521,10 @@ atomic_swap_32(volatile u_int32_t *p, u_int32_t v) #define atomic_fcmpset_rel_32 atomic_fcmpset_32 #define atomic_fcmpset_acq_32 atomic_fcmpset_32 #ifdef _KERNEL +#define atomic_fcmpset_8 atomic_fcmpset_8 #define atomic_fcmpset_rel_8 atomic_fcmpset_8 #define atomic_fcmpset_acq_8 atomic_fcmpset_8 +#define atomic_fcmpset_16 atomic_fcmpset_16 #define atomic_fcmpset_rel_16 atomic_fcmpset_16 #define atomic_fcmpset_acq_16 atomic_fcmpset_16 #define atomic_fcmpset_rel_64 atomic_fcmpset_64 @@ -533,8 +535,10 @@ atomic_swap_32(volatile u_int32_t *p, u_int32_t v) #define atomic_cmpset_rel_32 atomic_cmpset_32 #define atomic_cmpset_acq_32 atomic_cmpset_32 #ifdef _KERNEL +#define atomic_cmpset_8 atomic_cmpset_8 #define atomic_cmpset_rel_8 atomic_cmpset_8 #define atomic_cmpset_acq_8 atomic_cmpset_8 +#define atomic_cmpset_16 atomic_cmpset_16 #define atomic_cmpset_rel_16 atomic_cmpset_16 #define atomic_cmpset_acq_16 atomic_cmpset_16 #define atomic_cmpset_rel_64 atomic_cmpset_64 Modified: head/sys/arm/include/atomic-v6.h ============================================================================== --- head/sys/arm/include/atomic-v6.h Wed Mar 25 23:06:04 2020 (r359310) +++ head/sys/arm/include/atomic-v6.h Wed Mar 25 23:12:43 2020 (r359311) @@ -245,6 +245,7 @@ atomic_fcmpset_8(volatile uint8_t *_ptr, uint8_t *_old ATOMIC_FCMPSET_CODE(ret, uint8_t, "b"); return (ret); } +#define atomic_fcmpset_8 atomic_fcmpset_8 static __inline int atomic_fcmpset_acq_8(volatile uint8_t *_ptr, uint8_t *_old, uint8_t _new) @@ -274,6 +275,7 @@ atomic_fcmpset_16(volatile uint16_t *_ptr, uint16_t *_ ATOMIC_FCMPSET_CODE(ret, uint16_t, "h"); return (ret); } +#define atomic_fcmpset_16 atomic_fcmpset_16 static __inline int atomic_fcmpset_acq_16(volatile uint16_t *_ptr, uint16_t *_old, uint16_t _new) @@ -429,6 +431,7 @@ atomic_cmpset_8(volatile uint8_t *_ptr, uint8_t _old, ATOMIC_CMPSET_CODE(ret, "b"); return (ret); } +#define atomic_cmpset_8 atomic_cmpset_8 static __inline int atomic_cmpset_acq_8(volatile uint8_t *_ptr, uint8_t _old, uint8_t _new) @@ -458,6 +461,7 @@ atomic_cmpset_16(volatile uint16_t *_ptr, uint16_t _ol ATOMIC_CMPSET_CODE(ret, "h"); return (ret); } +#define atomic_cmpset_16 atomic_cmpset_16 static __inline int atomic_cmpset_acq_16(volatile uint16_t *_ptr, uint16_t _old, uint16_t _new) @@ -899,6 +903,7 @@ atomic_testandclear_long(volatile u_long *p, u_int v) return (atomic_testandclear_32((volatile uint32_t *)p, v)); } +#define atomic_testandclear_long atomic_testandclear_long static __inline int atomic_testandset_32(volatile uint32_t *ptr, u_int bit) @@ -942,6 +947,7 @@ atomic_testandset_long(volatile u_long *p, u_int v) return (atomic_testandset_32((volatile uint32_t *)p, v)); } +#define atomic_testandset_long atomic_testandset_long static __inline int atomic_testandset_64(volatile uint64_t *p, u_int v) Modified: head/sys/arm/include/atomic.h ============================================================================== --- head/sys/arm/include/atomic.h Wed Mar 25 23:06:04 2020 (r359310) +++ head/sys/arm/include/atomic.h Wed Mar 25 23:12:43 2020 (r359311) @@ -103,4 +103,6 @@ atomic_swap_long(volatile u_long *p, u_long v) #define atomic_store_rel_int atomic_store_rel_32 #define atomic_swap_int atomic_swap_32 +#include <sys/_atomic_subword.h> + #endif /* _MACHINE_ATOMIC_H_ */ Modified: head/sys/arm64/include/atomic.h ============================================================================== --- head/sys/arm64/include/atomic.h Wed Mar 25 23:06:04 2020 (r359310) +++ head/sys/arm64/include/atomic.h Wed Mar 25 23:12:43 2020 (r359311) @@ -257,6 +257,11 @@ _ATOMIC_FCMPSET_PROTO(t, bar, ) \ _ATOMIC_CMPSET_IMPL(32, w, , bar, a, l) \ _ATOMIC_CMPSET_IMPL(64, , , bar, a, l) +#define atomic_cmpset_8 atomic_cmpset_8 +#define atomic_fcmpset_8 atomic_fcmpset_8 +#define atomic_cmpset_16 atomic_cmpset_16 +#define atomic_fcmpset_16 atomic_fcmpset_16 + _ATOMIC_CMPSET( , , ) _ATOMIC_CMPSET(acq_, a, ) _ATOMIC_CMPSET(rel_, ,l) @@ -465,6 +470,8 @@ atomic_load_acq_##t(volatile uint##t##_t *p) \ return (ret); \ } +#define atomic_load_acq_8 atomic_load_acq_8 +#define atomic_load_acq_16 atomic_load_acq_16 _ATOMIC_LOAD_ACQ_IMPL(8, w, b) _ATOMIC_LOAD_ACQ_IMPL(16, w, h) _ATOMIC_LOAD_ACQ_IMPL(32, w, ) @@ -595,6 +602,8 @@ atomic_thread_fence_seq_cst(void) dmb(sy); } + +#include <sys/_atomic_subword.h> #endif /* KCSAN && !KCSAN_RUNTIME */ #endif /* _MACHINE_ATOMIC_H_ */ Modified: head/sys/i386/include/atomic.h ============================================================================== --- head/sys/i386/include/atomic.h Wed Mar 25 23:06:04 2020 (r359310) +++ head/sys/i386/include/atomic.h Wed Mar 25 23:12:43 2020 (r359311) @@ -808,6 +808,7 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_readandclear_int(p) atomic_swap_int(p, 0) #define atomic_readandclear_long(p) atomic_swap_long(p, 0) +#define atomic_testandset_acq_long atomic_testandset_long /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char Modified: head/sys/kern/subr_csan.c ============================================================================== --- head/sys/kern/subr_csan.c Wed Mar 25 23:06:04 2020 (r359310) +++ head/sys/kern/subr_csan.c Wed Mar 25 23:12:43 2020 (r359311) @@ -538,7 +538,7 @@ CSAN_ATOMIC_FUNC_ADD(8, uint8_t) CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t) CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t) CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t) -_CSAN_ATOMIC_FUNC_LOAD(8, uint8_t) +CSAN_ATOMIC_FUNC_LOAD(8, uint8_t) CSAN_ATOMIC_FUNC_SET(8, uint8_t) CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t) _CSAN_ATOMIC_FUNC_STORE(8, uint8_t) @@ -554,11 +554,7 @@ CSAN_ATOMIC_FUNC_ADD(16, uint16_t) CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t) CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t) CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t) -#if defined(__aarch64__) -_CSAN_ATOMIC_FUNC_LOAD(16, uint16_t) -#else CSAN_ATOMIC_FUNC_LOAD(16, uint16_t) -#endif CSAN_ATOMIC_FUNC_SET(16, uint16_t) CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t) _CSAN_ATOMIC_FUNC_STORE(16, uint16_t) @@ -632,6 +628,7 @@ CSAN_ATOMIC_FUNC_SWAP(long, u_long) #if !defined(__aarch64__) CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long) CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long) +CSAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long) #endif CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t) Modified: head/sys/powerpc/include/atomic.h ============================================================================== --- head/sys/powerpc/include/atomic.h Wed Mar 25 23:06:04 2020 (r359310) +++ head/sys/powerpc/include/atomic.h Wed Mar 25 23:12:43 2020 (r359311) @@ -727,11 +727,15 @@ ATOMIC_CMPSET_ACQ_REL(int); ATOMIC_CMPSET_ACQ_REL(long); +#ifdef ISA_206_ATOMICS #define atomic_cmpset_8 atomic_cmpset_char +#endif #define atomic_cmpset_acq_8 atomic_cmpset_acq_char #define atomic_cmpset_rel_8 atomic_cmpset_rel_char +#ifdef ISA_206_ATOMICS #define atomic_cmpset_16 atomic_cmpset_short +#endif #define atomic_cmpset_acq_16 atomic_cmpset_acq_short #define atomic_cmpset_rel_16 atomic_cmpset_rel_short @@ -894,11 +898,15 @@ atomic_fcmpset_long(volatile u_long *p, u_long *cmpval ATOMIC_FCMPSET_ACQ_REL(int); ATOMIC_FCMPSET_ACQ_REL(long); +#ifdef ISA_206_ATOMICS #define atomic_fcmpset_8 atomic_fcmpset_char +#endif #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char +#ifdef ISA_206_ATOMICS #define atomic_fcmpset_16 atomic_fcmpset_short +#endif #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short @@ -1018,6 +1026,10 @@ atomic_thread_fence_seq_cst(void) #ifndef ISA_206_ATOMICS #include <sys/_atomic_subword.h> +#define atomic_cmpset_char atomic_cmpset_8 +#define atomic_cmpset_short atomic_cmpset_16 +#define atomic_fcmpset_char atomic_fcmpset_8 +#define atomic_fcmpset_short atomic_fcmpset_16 #endif /* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */ Modified: head/sys/sys/_atomic_subword.h ============================================================================== --- head/sys/sys/_atomic_subword.h Wed Mar 25 23:06:04 2020 (r359310) +++ head/sys/sys/_atomic_subword.h Wed Mar 25 23:12:43 2020 (r359311) @@ -41,6 +41,9 @@ #endif #include <machine/endian.h> +#ifndef _KERNEL +#include <stdbool.h> +#endif #ifndef NBBY #define NBBY 8 @@ -113,6 +116,7 @@ _atomic_fcmpset_masked_word(uint32_t *addr, uint32_t * } #endif +#ifndef atomic_cmpset_8 static __inline int atomic_cmpset_8(__volatile uint8_t *addr, uint8_t old, uint8_t val) { @@ -123,7 +127,9 @@ atomic_cmpset_8(__volatile uint8_t *addr, uint8_t old, return (_atomic_cmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr), old << shift, val << shift, 0xff << shift)); } +#endif +#ifndef atomic_fcmpset_8 static __inline int atomic_fcmpset_8(__volatile uint8_t *addr, uint8_t *old, uint8_t val) { @@ -138,7 +144,9 @@ atomic_fcmpset_8(__volatile uint8_t *addr, uint8_t *ol *old = (wold >> shift) & 0xff; return (ret); } +#endif +#ifndef atomic_cmpset_16 static __inline int atomic_cmpset_16(__volatile uint16_t *addr, uint16_t old, uint16_t val) { @@ -149,7 +157,9 @@ atomic_cmpset_16(__volatile uint16_t *addr, uint16_t o return (_atomic_cmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr), old << shift, val << shift, 0xffff << shift)); } +#endif +#ifndef atomic_fcmpset_16 static __inline int atomic_fcmpset_16(__volatile uint16_t *addr, uint16_t *old, uint16_t val) { @@ -164,9 +174,101 @@ atomic_fcmpset_16(__volatile uint16_t *addr, uint16_t *old = (wold >> shift) & 0xffff; return (ret); } +#endif +#ifndef atomic_load_acq_8 +static __inline uint8_t +atomic_load_acq_8(volatile uint8_t *p) +{ + int shift; + uint8_t ret; + + shift = _ATOMIC_BYTE_SHIFT(p); + ret = (atomic_load_acq_32(_ATOMIC_WORD_ALIGNED(p)) >> shift) & 0xff; + return (ret); +} +#endif + +#ifndef atomic_load_acq_16 +static __inline uint16_t +atomic_load_acq_16(volatile uint16_t *p) +{ + int shift; + uint16_t ret; + + shift = _ATOMIC_HWORD_SHIFT(p); + ret = (atomic_load_acq_32(_ATOMIC_WORD_ALIGNED(p)) >> shift) & + 0xffff; + return (ret); +} +#endif + #undef _ATOMIC_WORD_ALIGNED #undef _ATOMIC_BYTE_SHIFT #undef _ATOMIC_HWORD_SHIFT + +/* + * Provide generic testandset_long implementation based on fcmpset long + * primitive. It may not be ideal for any given arch, so machine/atomic.h + * should define the macro atomic_testandset_long to override with an + * MD-specific version. + * + * (Organizationally, this isn't really subword atomics. But atomic_common is + * included too early in machine/atomic.h, so it isn't a good place for derived + * primitives like this.) + */ +#ifndef atomic_testandset_acq_long +static __inline int +atomic_testandset_acq_long(volatile u_long *p, u_int v) +{ + u_long bit, old; + bool ret; + + bit = (1ul << (v % (sizeof(*p) * NBBY))); + + old = atomic_load_acq_long(p); + ret = false; + while (!ret && (old & bit) == 0) + ret = atomic_fcmpset_acq_long(p, &old, old | bit); + + return (!ret); +} +#endif + +#ifndef atomic_testandset_long +static __inline int +atomic_testandset_long(volatile u_long *p, u_int v) +{ + u_long bit, old; + bool ret; + + bit = (1ul << (v % (sizeof(*p) * NBBY))); + + old = atomic_load_long(p); + ret = false; + while (!ret && (old & bit) == 0) + ret = atomic_fcmpset_long(p, &old, old | bit); + + return (!ret); +} +#endif + +#ifndef atomic_testandclear_long +static __inline int +atomic_testandclear_long(volatile u_long *p, u_int v) +{ + u_long bit, old; + bool ret; + + bit = (1ul << (v % (sizeof(*p) * NBBY))); + + old = atomic_load_long(p); + ret = false; + while (!ret && (old & bit) != 0) + ret = atomic_fcmpset_long(p, &old, old & ~bit); + + return (ret); +} +#endif #endif /* _SYS__ATOMIC_SUBWORD_H_ */ Modified: head/sys/sys/_cscan_atomic.h ============================================================================== --- head/sys/sys/_cscan_atomic.h Wed Mar 25 23:06:04 2020 (r359310) +++ head/sys/sys/_cscan_atomic.h Wed Mar 25 23:12:43 2020 (r359311) @@ -69,7 +69,8 @@ void kcsan_atomic_store_rel_##name(volatile type *, type) #define KCSAN_ATOMIC_TEST(op, name, type) \ - int kcsan_atomic_##op##_##name(volatile type *, u_int) + int kcsan_atomic_##op##_##name(volatile type *, u_int); \ + int kcsan_atomic_##op##_acq_##name(volatile type *, u_int) #define KCSAN_ATOMIC_FUNCS(name, type) \ KCSAN_ATOMIC_FUNC_1(add, name, type); \ @@ -156,6 +157,7 @@ void kcsan_atomic_thread_fence_seq_cst(void); #define atomic_swap_long kcsan_atomic_swap_long #define atomic_testandclear_long kcsan_atomic_testandclear_long #define atomic_testandset_long kcsan_atomic_testandset_long +#define atomic_testandset_acq_long kcsan_atomic_testandset_acq_long #define atomic_add_ptr kcsan_atomic_add_ptr #define atomic_add_acq_ptr kcsan_atomic_add_acq_ptr
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202003252312.02PNChC7051461>