From owner-svn-src-head@freebsd.org Mon Feb 3 18:23:14 2020 Return-Path: Delivered-To: svn-src-head@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id E74EF22A48B; Mon, 3 Feb 2020 18:23:14 +0000 (UTC) (envelope-from markj@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) server-signature RSA-PSS (4096 bits) client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 48BGR65vSwz3NtK; Mon, 3 Feb 2020 18:23:14 +0000 (UTC) (envelope-from markj@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id C5D1F4F55; Mon, 3 Feb 2020 18:23:14 +0000 (UTC) (envelope-from markj@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id 013INElk024703; Mon, 3 Feb 2020 18:23:14 GMT (envelope-from markj@FreeBSD.org) Received: (from markj@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id 013INEWY024702; Mon, 3 Feb 2020 18:23:14 GMT (envelope-from markj@FreeBSD.org) Message-Id: <202002031823.013INEWY024702@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: markj set sender to markj@FreeBSD.org using -f From: Mark Johnston Date: Mon, 3 Feb 2020 18:23:14 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r357458 - head/sys/arm64/include X-SVN-Group: head X-SVN-Commit-Author: markj X-SVN-Commit-Paths: head/sys/arm64/include X-SVN-Commit-Revision: 357458 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 03 Feb 2020 18:23:15 -0000 Author: markj Date: Mon Feb 3 18:23:14 2020 New Revision: 357458 URL: https://svnweb.freebsd.org/changeset/base/357458 Log: Add wrappers for arm64 atomics. Add a _llsc suffix for the existing LL/SC-based implementations and add trivial wrappers. This is in preparation for supporting LSE-based atomic(9) implementations. No functional change intended. Reviewed by: andrew, kib MFC after: 1 month Sponsored by: The FreeBSD Foundation, Amazon (hardware) Differential Revision: https://reviews.freebsd.org/D23323 Modified: head/sys/arm64/include/atomic.h Modified: head/sys/arm64/include/atomic.h ============================================================================== --- head/sys/arm64/include/atomic.h Mon Feb 3 18:22:59 2020 (r357457) +++ head/sys/arm64/include/atomic.h Mon Feb 3 18:23:14 2020 (r357458) @@ -59,9 +59,12 @@ #include -#define _ATOMIC_OP_IMPL(t, w, s, op, asm_op, bar, a, l) \ +#define _ATOMIC_OP_PROTO(t, op, bar, flav) \ static __inline void \ -atomic_##op##_##bar##t(volatile uint##t##_t *p, uint##t##_t val) \ +atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val) + +#define _ATOMIC_OP_IMPL(t, w, s, op, asm_op, bar, a, l) \ +_ATOMIC_OP_PROTO(t, op, bar, _llsc) \ { \ uint##t##_t tmp; \ int res; \ @@ -75,6 +78,11 @@ atomic_##op##_##bar##t(volatile uint##t##_t *p, uint## : "r" (p), "r" (val) \ : "memory" \ ); \ +} \ + \ +_ATOMIC_OP_PROTO(t, op, bar, ) \ +{ \ + atomic_##op##_##bar##t##_llsc(p, val); \ } #define __ATOMIC_OP(op, asm_op, bar, a, l) \ @@ -93,10 +101,18 @@ _ATOMIC_OP(clear, bic) _ATOMIC_OP(set, orr) _ATOMIC_OP(subtract, sub) -#define _ATOMIC_CMPSET_IMPL(t, w, s, bar, a, l) \ +#define _ATOMIC_CMPSET_PROTO(t, bar, flav) \ static __inline int \ -atomic_cmpset_##bar##t(volatile uint##t##_t *p, uint##t##_t cmpval, \ - uint##t##_t newval) \ +atomic_cmpset_##bar##t##flav(volatile uint##t##_t *p, \ + uint##t##_t cmpval, uint##t##_t newval) + +#define _ATOMIC_FCMPSET_PROTO(t, bar, flav) \ +static __inline int \ +atomic_fcmpset_##bar##t##flav(volatile uint##t##_t *p, \ + uint##t##_t *cmpval, uint##t##_t newval) + +#define _ATOMIC_CMPSET_IMPL(t, w, s, bar, a, l) \ +_ATOMIC_CMPSET_PROTO(t, bar, _llsc) \ { \ uint##t##_t tmp; \ int res; \ @@ -117,10 +133,13 @@ atomic_cmpset_##bar##t(volatile uint##t##_t *p, uint## return (!res); \ } \ \ -static __inline int \ -atomic_fcmpset_##bar##t(volatile uint##t##_t *p, uint##t##_t *cmpval, \ - uint##t##_t newval) \ +_ATOMIC_CMPSET_PROTO(t, bar, ) \ { \ + return (atomic_cmpset_##bar##t##_llsc(p, cmpval, newval)); \ +} \ + \ +_ATOMIC_FCMPSET_PROTO(t, bar, _llsc) \ +{ \ uint##t##_t _cmpval, tmp; \ int res; \ \ @@ -139,6 +158,11 @@ atomic_fcmpset_##bar##t(volatile uint##t##_t *p, uint# *cmpval = tmp; \ \ return (!res); \ +} \ + \ +_ATOMIC_FCMPSET_PROTO(t, bar, ) \ +{ \ + return (atomic_fcmpset_##bar##t##_llsc(p, cmpval, newval)); \ } #define _ATOMIC_CMPSET(bar, a, l) \ @@ -151,9 +175,12 @@ _ATOMIC_CMPSET( , , ) _ATOMIC_CMPSET(acq_, a, ) _ATOMIC_CMPSET(rel_, ,l) -#define _ATOMIC_FETCHADD_IMPL(t, w) \ +#define _ATOMIC_FETCHADD_PROTO(t, flav) \ static __inline uint##t##_t \ -atomic_fetchadd_##t(volatile uint##t##_t *p, uint##t##_t val) \ +atomic_fetchadd_##t##flav(volatile uint##t##_t *p, uint##t##_t val) + +#define _ATOMIC_FETCHADD_IMPL(t, w) \ +_ATOMIC_FETCHADD_PROTO(t, _llsc) \ { \ uint##t##_t tmp, ret; \ int res; \ @@ -169,14 +196,26 @@ atomic_fetchadd_##t(volatile uint##t##_t *p, uint##t## ); \ \ return (ret); \ +} \ + \ +_ATOMIC_FETCHADD_PROTO(t, ) \ +{ \ + return (atomic_fetchadd_##t##_llsc(p, val)); \ } _ATOMIC_FETCHADD_IMPL(32, w) _ATOMIC_FETCHADD_IMPL(64, ) -#define _ATOMIC_SWAP_IMPL(t, w, zreg) \ +#define _ATOMIC_SWAP_PROTO(t, flav) \ static __inline uint##t##_t \ -atomic_swap_##t(volatile uint##t##_t *p, uint##t##_t val) \ +atomic_swap_##t##flav(volatile uint##t##_t *p, uint##t##_t val) + +#define _ATOMIC_READANDCLEAR_PROTO(t, flav) \ +static __inline uint##t##_t \ +atomic_readandclear_##t##flav(volatile uint##t##_t *p) + +#define _ATOMIC_SWAP_IMPL(t, w, zreg) \ +_ATOMIC_SWAP_PROTO(t, _llsc) \ { \ uint##t##_t ret; \ int res; \ @@ -193,9 +232,13 @@ atomic_swap_##t(volatile uint##t##_t *p, uint##t##_t v return (ret); \ } \ \ -static __inline uint##t##_t \ -atomic_readandclear_##t(volatile uint##t##_t *p) \ +_ATOMIC_SWAP_PROTO(t, ) \ { \ + return (atomic_swap_##t##_llsc(p, val)); \ +} \ + \ +_ATOMIC_READANDCLEAR_PROTO(t, _llsc) \ +{ \ uint##t##_t ret; \ int res; \ \ @@ -209,14 +252,22 @@ atomic_readandclear_##t(volatile uint##t##_t *p) \ ); \ \ return (ret); \ +} \ + \ +_ATOMIC_READANDCLEAR_PROTO(t, ) \ +{ \ + return (atomic_readandclear_##t##_llsc(p)); \ } _ATOMIC_SWAP_IMPL(32, w, wzr) _ATOMIC_SWAP_IMPL(64, , xzr) -#define _ATOMIC_TEST_OP_IMPL(t, w, op, asm_op) \ +#define _ATOMIC_TEST_OP_PROTO(t, op, flav) \ static __inline int \ -atomic_testand##op##_##t(volatile uint##t##_t *p, u_int val) \ +atomic_testand##op##_##t##flav(volatile uint##t##_t *p, u_int val) + +#define _ATOMIC_TEST_OP_IMPL(t, w, op, asm_op) \ +_ATOMIC_TEST_OP_PROTO(t, op, _llsc) \ { \ uint##t##_t mask, old, tmp; \ int res; \ @@ -233,6 +284,11 @@ atomic_testand##op##_##t(volatile uint##t##_t *p, u_in ); \ \ return ((old & mask) != 0); \ +} \ + \ +_ATOMIC_TEST_OP_PROTO(t, op, ) \ +{ \ + return (atomic_testand##op##_##t##_llsc(p, val)); \ } #define _ATOMIC_TEST_OP(op, asm_op) \ @@ -389,5 +445,4 @@ atomic_thread_fence_seq_cst(void) } #endif /* KCSAN && !KCSAN_RUNTIME */ - #endif /* _MACHINE_ATOMIC_H_ */