Date: Wed, 6 Oct 2021 18:53:17 GMT From: Kristof Provost <kp@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org Subject: git: 92a6b17e8ba0 - stable/12 - [PowerPC] [MIPS] Implement 32-bit kernel emulation of atomic64 operations Message-ID: <202110061853.196IrH9L074246@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch stable/12 has been updated by kp: URL: https://cgit.FreeBSD.org/src/commit/?id=92a6b17e8ba00d1d55936b53868f81c69d70f446 commit 92a6b17e8ba00d1d55936b53868f81c69d70f446 Author: Brandon Bergren <bdragon@FreeBSD.org> AuthorDate: 2020-01-02 23:20:37 +0000 Commit: Kristof Provost <kp@FreeBSD.org> CommitDate: 2021-10-06 18:29:31 +0000 [PowerPC] [MIPS] Implement 32-bit kernel emulation of atomic64 operations This is a lock-based emulation of 64-bit atomics for kernel use, split off from an earlier patch by jhibbits. This is needed to unblock future improvements that reduce the need for locking on 64-bit platforms by using atomic updates. The implementation allows for future integration with userland atomic64, but as that implies going through sysarch for every use, the current status quo of userland doing its own locking may be for the best. Submitted by: jhibbits (original patch), kevans (mips bits) Reviewed by: jhibbits, jeff, kevans Differential Revision: https://reviews.freebsd.org/D22976 (cherry picked from commit 9aafc7c0523456c8b5ee8919c97f75277cf4d70b) --- .../compat/opensolaris/kern/opensolaris_atomic.c | 3 +- sys/cddl/compat/opensolaris/sys/atomic.h | 6 +- sys/conf/files.mips | 1 + sys/conf/files.powerpc | 1 + sys/kern/subr_atomic64.c | 140 +++++++++++++++++++++ sys/mips/include/atomic.h | 4 + sys/powerpc/include/atomic.h | 4 + sys/sys/_atomic64e.h | 80 ++++++++++++ 8 files changed, 236 insertions(+), 3 deletions(-) diff --git a/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c b/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c index 89cd1db46742..99a79b306580 100644 --- a/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c +++ b/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c @@ -33,7 +33,8 @@ __FBSDID("$FreeBSD$"); #include <sys/atomic.h> #if !defined(__LP64__) && !defined(__mips_n32) && \ - !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) + !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) && \ + !defined(HAS_EMULATED_ATOMIC64) #ifdef _KERNEL #include <sys/kernel.h> diff --git a/sys/cddl/compat/opensolaris/sys/atomic.h b/sys/cddl/compat/opensolaris/sys/atomic.h index 9e30f3da6669..b3a2f393afa4 100644 --- a/sys/cddl/compat/opensolaris/sys/atomic.h +++ b/sys/cddl/compat/opensolaris/sys/atomic.h @@ -42,7 +42,8 @@ #endif #if !defined(__LP64__) && !defined(__mips_n32) && \ - !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) + !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64) && \ + !defined(HAS_EMULATED_ATOMIC64) extern void atomic_add_64(volatile uint64_t *target, int64_t delta); extern void atomic_dec_64(volatile uint64_t *target); extern uint64_t atomic_swap_64(volatile uint64_t *a, uint64_t value); @@ -109,7 +110,8 @@ atomic_cas_32(volatile uint32_t *target, uint32_t cmp, uint32_t newval) #endif #if defined(__LP64__) || defined(__mips_n32) || \ - defined(ARM_HAVE_ATOMIC64) || defined(I386_HAVE_ATOMIC64) + defined(ARM_HAVE_ATOMIC64) || defined(I386_HAVE_ATOMIC64) || \ + defined(HAS_EMULATED_ATOMIC64) static __inline void atomic_dec_64(volatile uint64_t *target) { diff --git a/sys/conf/files.mips b/sys/conf/files.mips index 1977fb9dcce2..4781c9adb4bb 100644 --- a/sys/conf/files.mips +++ b/sys/conf/files.mips @@ -50,6 +50,7 @@ mips/mips/vm_machdep.c standard # misc opt-in bits kern/kern_clocksource.c standard kern/link_elf_obj.c standard +kern/subr_atomic64.c optional mips | mipsel | mipshf | mipselhf kern/subr_busdma_bufalloc.c standard kern/subr_dummy_vdso_tc.c standard kern/subr_sfbuf.c optional mips | mipsel | mipsn32 diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc index eb55f955ebaa..90f683f7a890 100644 --- a/sys/conf/files.powerpc +++ b/sys/conf/files.powerpc @@ -120,6 +120,7 @@ dev/uart/uart_cpu_powerpc.c optional uart dev/usb/controller/ehci_fsl.c optional ehci mpc85xx dev/vt/hw/ofwfb/ofwfb.c optional vt aim kern/kern_clocksource.c standard +kern/subr_atomic64.c optional powerpc | powerpcspe kern/subr_dummy_vdso_tc.c standard kern/syscalls.c optional ktr kern/subr_sfbuf.c standard diff --git a/sys/kern/subr_atomic64.c b/sys/kern/subr_atomic64.c new file mode 100644 index 000000000000..266b4ee20a10 --- /dev/null +++ b/sys/kern/subr_atomic64.c @@ -0,0 +1,140 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 Justin Hibbits + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include <sys/cdefs.h> +#include <sys/param.h> +#include <sys/kernel.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/smp.h> +#include <sys/systm.h> + +#include <machine/atomic.h> +#include <machine/param.h> + +#include <vm/vm.h> +#include <vm/pmap.h> + +enum { + ATOMIC64_ADD, + ATOMIC64_CLEAR, + ATOMIC64_CMPSET, + ATOMIC64_FCMPSET, + ATOMIC64_FETCHADD, + ATOMIC64_LOAD, + ATOMIC64_SET, + ATOMIC64_SUBTRACT, + ATOMIC64_STORE, + ATOMIC64_SWAP +}; + +#ifdef _KERNEL +#define A64_POOL_SIZE MAXCPU +/* Estimated size of a cacheline */ +#define CACHE_ALIGN CACHE_LINE_SIZE + +#define GET_MUTEX(p) \ + (&a64_mtx_pool[(pmap_kextract((vm_offset_t)p) / CACHE_ALIGN) % (A64_POOL_SIZE)]) + +#define LOCK_A64() \ + struct mtx *_amtx = GET_MUTEX(p); \ + if (smp_started) mtx_lock(_amtx) + +#define UNLOCK_A64() if (smp_started) mtx_unlock(_amtx) + +#define ATOMIC64_EMU_UN(op, rt, block, ret) \ + rt \ + atomic_##op##_64(volatile u_int64_t *p) { \ + u_int64_t tmp __unused; \ + LOCK_A64(); \ + block; \ + UNLOCK_A64(); \ + ret; } struct hack + +#define ATOMIC64_EMU_BIN(op, rt, block, ret) \ + rt \ + atomic_##op##_64(volatile u_int64_t *p, u_int64_t v) { \ + u_int64_t tmp __unused; \ + LOCK_A64(); \ + block; \ + UNLOCK_A64(); \ + ret; } struct hack + +static struct mtx a64_mtx_pool[A64_POOL_SIZE]; + +ATOMIC64_EMU_BIN(add, void, (*p = *p + v), return); +ATOMIC64_EMU_BIN(clear, void, *p &= ~v, return); +ATOMIC64_EMU_BIN(fetchadd, u_int64_t, (*p = *p + v, v = *p - v), return (v)); +ATOMIC64_EMU_UN(load, u_int64_t, (tmp = *p), return (tmp)); +ATOMIC64_EMU_BIN(set, void, *p |= v, return); +ATOMIC64_EMU_BIN(subtract, void, (*p = *p - v), return); +ATOMIC64_EMU_BIN(store, void, *p = v, return); +ATOMIC64_EMU_BIN(swap, u_int64_t, tmp = *p; *p = v; v = tmp, return(v)); + +int atomic_cmpset_64(volatile u_int64_t *p, u_int64_t old, u_int64_t new) +{ + u_int64_t tmp; + + LOCK_A64(); + tmp = *p; + if (tmp == old) + *p = new; + UNLOCK_A64(); + + return (tmp == old); +} + +int atomic_fcmpset_64(volatile u_int64_t *p, u_int64_t *old, u_int64_t new) +{ + u_int64_t tmp, tmp_old; + + LOCK_A64(); + tmp = *p; + tmp_old = *old; + if (tmp == tmp_old) + *p = new; + else + *old = tmp; + UNLOCK_A64(); + + return (tmp == tmp_old); +} + +static void +atomic64_mtxinit(void *x __unused) +{ + int i; + + for (i = 0; i < A64_POOL_SIZE; i++) + mtx_init(&a64_mtx_pool[i], "atomic64 mutex", NULL, MTX_DEF); +} + +SYSINIT(atomic64_mtxinit, SI_SUB_LOCK, SI_ORDER_MIDDLE, atomic64_mtxinit, NULL); + +#endif /* _KERNEL */ diff --git a/sys/mips/include/atomic.h b/sys/mips/include/atomic.h index 709af5cdcf59..0ede70d82202 100644 --- a/sys/mips/include/atomic.h +++ b/sys/mips/include/atomic.h @@ -38,6 +38,10 @@ #include <sys/atomic_common.h> +#if !defined(__mips_n64) && !defined(__mips_n32) +#include <sys/_atomic64e.h> +#endif + /* * Note: All the 64-bit atomic operations are only atomic when running * in 64-bit mode. It is assumed that code compiled for n32 and n64 diff --git a/sys/powerpc/include/atomic.h b/sys/powerpc/include/atomic.h index afffdd7bb4ed..e49a8473a747 100644 --- a/sys/powerpc/include/atomic.h +++ b/sys/powerpc/include/atomic.h @@ -40,6 +40,10 @@ #include <sys/atomic_common.h> +#ifndef __powerpc64__ +#include <sys/_atomic64e.h> +#endif + /* * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction * with the atomic lXarx/stXcx. sequences below. They are not exposed outside diff --git a/sys/sys/_atomic64e.h b/sys/sys/_atomic64e.h new file mode 100644 index 000000000000..289e5117ee0b --- /dev/null +++ b/sys/sys/_atomic64e.h @@ -0,0 +1,80 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 Justin Hibbits + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + + +#ifndef _SYS_ATOMIC64E_H_ +#define _SYS_ATOMIC64E_H_ + +#ifndef _MACHINE_ATOMIC_H_ +#error "This should not be included directly. Include <machine/atomic.h>" +#endif + +#ifdef _KERNEL +#define HAS_EMULATED_ATOMIC64 + +/* Emulated versions of 64-bit atomic operations. */ + +void atomic_add_64(volatile u_int64_t *, u_int64_t); +#define atomic_add_acq_64 atomic_add_64 +#define atomic_add_rel_64 atomic_add_64 + +int atomic_cmpset_64(volatile u_int64_t *, u_int64_t, u_int64_t); +#define atomic_cmpset_acq_64 atomic_cmpset_64 +#define atomic_cmpset_rel_64 atomic_cmpset_64 + +void atomic_clear_64(volatile u_int64_t *, u_int64_t); +#define atomic_clear_acq_64 atomic_clear_64 +#define atomic_clear_rel_64 atomic_clear_64 + +int atomic_fcmpset_64(volatile u_int64_t *, u_int64_t *, u_int64_t); +#define atomic_fcmpset_acq_64 atomic_fcmpset_64 +#define atomic_fcmpset_rel_64 atomic_fcmpset_64 + +u_int64_t atomic_fetchadd_64(volatile u_int64_t *, u_int64_t); + +u_int64_t atomic_load_64(volatile u_int64_t *); +#define atomic_load_acq_64 atomic_load_64 + +void atomic_readandclear_64(volatile u_int64_t *); + +void atomic_set_64(volatile u_int64_t *, u_int64_t); +#define atomic_set_acq_64 atomic_set_64 +#define atomic_set_rel_64 atomic_set_64 + +void atomic_subtract_64(volatile u_int64_t *, u_int64_t); +#define atomic_subtract_acq_64 atomic_subtract_64 +#define atomic_subtract_rel_64 atomic_subtract_64 + +void atomic_store_64(volatile u_int64_t *, u_int64_t); +#define atomic_store_rel_64 atomic_store_64 + +u_int64_t atomic_swap_64(volatile u_int64_t *, u_int64_t); + +#endif /* _KERNEL */ +#endif /* _SYS_ATOMIC64E_H_ */
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202110061853.196IrH9L074246>