From owner-svn-src-head@freebsd.org Tue Oct 8 01:36:34 2019 Return-Path: Delivered-To: svn-src-head@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id F2653121E2E; Tue, 8 Oct 2019 01:36:34 +0000 (UTC) (envelope-from jhibbits@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) server-signature RSA-PSS (4096 bits) client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 46nKh25z5lz3ylq; Tue, 8 Oct 2019 01:36:34 +0000 (UTC) (envelope-from jhibbits@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id B070920913; Tue, 8 Oct 2019 01:36:34 +0000 (UTC) (envelope-from jhibbits@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id x981aY21073146; Tue, 8 Oct 2019 01:36:34 GMT (envelope-from jhibbits@FreeBSD.org) Received: (from jhibbits@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id x981aYTq073145; Tue, 8 Oct 2019 01:36:34 GMT (envelope-from jhibbits@FreeBSD.org) Message-Id: <201910080136.x981aYTq073145@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: jhibbits set sender to jhibbits@FreeBSD.org using -f From: Justin Hibbits Date: Tue, 8 Oct 2019 01:36:34 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r353296 - head/sys/powerpc/include X-SVN-Group: head X-SVN-Commit-Author: jhibbits X-SVN-Commit-Paths: head/sys/powerpc/include X-SVN-Commit-Revision: 353296 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 08 Oct 2019 01:36:35 -0000 Author: jhibbits Date: Tue Oct 8 01:36:34 2019 New Revision: 353296 URL: https://svnweb.freebsd.org/changeset/base/353296 Log: powerpc: Implement atomic_(f)cmpset_ for short and char | This adds two implementations for each atomic_fcmpset_ and atomic_cmpset_ short and char functions, selectable at compile time for the target architecture. By default, it uses a generic shift-and-mask to perform atomic updates to sub-components of 32-bit words from . However, if ISA_206_ATOMICS is defined it uses the ll/sc instructions for halfword and bytes, introduced in PowerISA 2.06. These instructions are supported by all IBM processors from POWER7 on, as well as the Freescale/NXP e6500 core. Although the e5500 and e500mc both implement PowerISA 2.06 they do not implement these instructions. As part of this, clean up the atomic_(f)cmpset_acq and _rel wrappers, by using macros to reduce code duplication. ISA_206_ATOMICS requires clang or newer binutils (2.20 or later). Differential Revision: https://reviews.freebsd.org/D21682 Modified: head/sys/powerpc/include/atomic.h Modified: head/sys/powerpc/include/atomic.h ============================================================================== --- head/sys/powerpc/include/atomic.h Mon Oct 7 23:35:23 2019 (r353295) +++ head/sys/powerpc/include/atomic.h Tue Oct 8 01:36:34 2019 (r353296) @@ -560,7 +560,57 @@ atomic_store_rel_long(volatile u_long *addr, u_long va * two values are equal, update the value of *p with newval. Returns * zero if the compare failed, nonzero otherwise. */ +#ifdef ISA_206_ATOMICS static __inline int +atomic_cmpset_char(volatile u_char *p, u_char cmpval, u_char newval) +{ + int ret; + + __asm __volatile ( + "1:\tlbarx %0, 0, %2\n\t" /* load old value */ + "cmplw %3, %0\n\t" /* compare */ + "bne- 2f\n\t" /* exit if not equal */ + "stbcx. %4, 0, %2\n\t" /* attempt to store */ + "bne- 1b\n\t" /* spin if failed */ + "li %0, 1\n\t" /* success - retval = 1 */ + "b 3f\n\t" /* we've succeeded */ + "2:\n\t" + "stbcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ + "li %0, 0\n\t" /* failure - retval = 0 */ + "3:\n\t" + : "=&r" (ret), "=m" (*p) + : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) + : "cr0", "memory"); + + return (ret); +} + +static __inline int +atomic_cmpset_short(volatile u_short *p, u_short cmpval, u_short newval) +{ + int ret; + + __asm __volatile ( + "1:\tlharx %0, 0, %2\n\t" /* load old value */ + "cmplw %3, %0\n\t" /* compare */ + "bne- 2f\n\t" /* exit if not equal */ + "sthcx. %4, 0, %2\n\t" /* attempt to store */ + "bne- 1b\n\t" /* spin if failed */ + "li %0, 1\n\t" /* success - retval = 1 */ + "b 3f\n\t" /* we've succeeded */ + "2:\n\t" + "sthcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ + "li %0, 0\n\t" /* failure - retval = 0 */ + "3:\n\t" + : "=&r" (ret), "=m" (*p) + : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) + : "cr0", "memory"); + + return (ret); +} +#endif + +static __inline int atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) { int ret; @@ -618,40 +668,37 @@ atomic_cmpset_long(volatile u_long* p, u_long cmpval, return (ret); } -static __inline int -atomic_cmpset_acq_int(volatile u_int *p, u_int cmpval, u_int newval) -{ - int retval; +#define ATOMIC_CMPSET_ACQ_REL(type) \ + static __inline int \ + atomic_cmpset_acq_##type(volatile u_##type *p, \ + u_##type cmpval, u_##type newval)\ + {\ + u_##type retval; \ + retval = atomic_cmpset_##type(p, cmpval, newval);\ + __ATOMIC_ACQ();\ + return (retval);\ + }\ + static __inline int \ + atomic_cmpset_rel_##type(volatile u_##type *p, \ + u_##type cmpval, u_##type newval)\ + {\ + __ATOMIC_ACQ();\ + return (atomic_cmpset_##type(p, cmpval, newval));\ + }\ + struct hack - retval = atomic_cmpset_int(p, cmpval, newval); - __ATOMIC_ACQ(); - return (retval); -} +ATOMIC_CMPSET_ACQ_REL(int); +ATOMIC_CMPSET_ACQ_REL(long); -static __inline int -atomic_cmpset_rel_int(volatile u_int *p, u_int cmpval, u_int newval) -{ - __ATOMIC_REL(); - return (atomic_cmpset_int(p, cmpval, newval)); -} -static __inline int -atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval) -{ - u_long retval; +#define atomic_cmpset_8 atomic_cmpset_char +#define atomic_cmpset_acq_8 atomic_cmpset_acq_char +#define atomic_cmpset_rel_8 atomic_cmpset_rel_char - retval = atomic_cmpset_long(p, cmpval, newval); - __ATOMIC_ACQ(); - return (retval); -} +#define atomic_cmpset_16 atomic_cmpset_short +#define atomic_cmpset_acq_16 atomic_cmpset_acq_short +#define atomic_cmpset_rel_16 atomic_cmpset_rel_short -static __inline int -atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval) -{ - __ATOMIC_REL(); - return (atomic_cmpset_long(p, cmpval, newval)); -} - #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int @@ -676,13 +723,65 @@ atomic_cmpset_rel_long(volatile u_long *p, u_long cmpv * zero if the compare failed and sets *cmpval to the read value from *p, * nonzero otherwise. */ +#ifdef ISA_206_ATOMICS static __inline int +atomic_fcmpset_char(volatile u_char *p, u_char *cmpval, u_char newval) +{ + int ret; + + __asm __volatile ( + "lbarx %0, 0, %3\n\t" /* load old value */ + "cmplw %4, %0\n\t" /* compare */ + "bne- 1f\n\t" /* exit if not equal */ + "stbcx. %5, 0, %3\n\t" /* attempt to store */ + "bne- 1f\n\t" /* exit if failed */ + "li %0, 1\n\t" /* success - retval = 1 */ + "b 2f\n\t" /* we've succeeded */ + "1:\n\t" + "stbcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ + "stwx %0, 0, %7\n\t" + "li %0, 0\n\t" /* failure - retval = 0 */ + "2:\n\t" + : "=&r" (ret), "=m" (*p), "=m" (*cmpval) + : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) + : "cr0", "memory"); + + return (ret); +} + +static __inline int +atomic_fcmpset_short(volatile u_short *p, u_short *cmpval, u_short newval) +{ + int ret; + + __asm __volatile ( + "lharx %0, 0, %3\n\t" /* load old value */ + "cmplw %4, %0\n\t" /* compare */ + "bne- 1f\n\t" /* exit if not equal */ + "sthcx. %5, 0, %3\n\t" /* attempt to store */ + "bne- 1f\n\t" /* exit if failed */ + "li %0, 1\n\t" /* success - retval = 1 */ + "b 2f\n\t" /* we've succeeded */ + "1:\n\t" + "sthcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ + "stwx %0, 0, %7\n\t" + "li %0, 0\n\t" /* failure - retval = 0 */ + "2:\n\t" + : "=&r" (ret), "=m" (*p), "=m" (*cmpval) + : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) + : "cr0", "memory"); + + return (ret); +} +#endif /* ISA_206_ATOMICS */ + +static __inline int atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval) { int ret; __asm __volatile ( - "lwarx %0, 0, %3\n\t" /* load old value */ + "lwarx %0, 0, %3\n\t" /* load old value */ "cmplw %4, %0\n\t" /* compare */ "bne- 1f\n\t" /* exit if not equal */ "stwcx. %5, 0, %3\n\t" /* attempt to store */ @@ -707,12 +806,12 @@ atomic_fcmpset_long(volatile u_long *p, u_long *cmpval __asm __volatile ( #ifdef __powerpc64__ - "ldarx %0, 0, %3\n\t" /* load old value */ + "ldarx %0, 0, %3\n\t" /* load old value */ "cmpld %4, %0\n\t" /* compare */ "bne- 1f\n\t" /* exit if not equal */ "stdcx. %5, 0, %3\n\t" /* attempt to store */ #else - "lwarx %0, 0, %3\n\t" /* load old value */ + "lwarx %0, 0, %3\n\t" /* load old value */ "cmplw %4, %0\n\t" /* compare */ "bne- 1f\n\t" /* exit if not equal */ "stwcx. %5, 0, %3\n\t" /* attempt to store */ @@ -737,40 +836,36 @@ atomic_fcmpset_long(volatile u_long *p, u_long *cmpval return (ret); } -static __inline int -atomic_fcmpset_acq_int(volatile u_int *p, u_int *cmpval, u_int newval) -{ - int retval; +#define ATOMIC_FCMPSET_ACQ_REL(type) \ + static __inline int \ + atomic_fcmpset_acq_##type(volatile u_##type *p, \ + u_##type *cmpval, u_##type newval)\ + {\ + u_##type retval; \ + retval = atomic_fcmpset_##type(p, cmpval, newval);\ + __ATOMIC_ACQ();\ + return (retval);\ + }\ + static __inline int \ + atomic_fcmpset_rel_##type(volatile u_##type *p, \ + u_##type *cmpval, u_##type newval)\ + {\ + __ATOMIC_REL();\ + return (atomic_fcmpset_##type(p, cmpval, newval));\ + }\ + struct hack - retval = atomic_fcmpset_int(p, cmpval, newval); - __ATOMIC_ACQ(); - return (retval); -} +ATOMIC_FCMPSET_ACQ_REL(int); +ATOMIC_FCMPSET_ACQ_REL(long); -static __inline int -atomic_fcmpset_rel_int(volatile u_int *p, u_int *cmpval, u_int newval) -{ - __ATOMIC_REL(); - return (atomic_fcmpset_int(p, cmpval, newval)); -} +#define atomic_fcmpset_8 atomic_fcmpset_char +#define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char +#define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char -static __inline int -atomic_fcmpset_acq_long(volatile u_long *p, u_long *cmpval, u_long newval) -{ - u_long retval; +#define atomic_fcmpset_16 atomic_fcmpset_short +#define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short +#define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short - retval = atomic_fcmpset_long(p, cmpval, newval); - __ATOMIC_ACQ(); - return (retval); -} - -static __inline int -atomic_fcmpset_rel_long(volatile u_long *p, u_long *cmpval, u_long newval) -{ - __ATOMIC_REL(); - return (atomic_fcmpset_long(p, cmpval, newval)); -} - #define atomic_fcmpset_32 atomic_fcmpset_int #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int @@ -857,9 +952,6 @@ atomic_swap_64(volatile u_long *p, u_long v) #define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v) #endif -#undef __ATOMIC_REL -#undef __ATOMIC_ACQ - static __inline void atomic_thread_fence_acq(void) { @@ -887,5 +979,19 @@ atomic_thread_fence_seq_cst(void) __asm __volatile("sync" : : : "memory"); } + +#ifndef ISA_206_ATOMICS +#include +#endif + +/* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */ +ATOMIC_CMPSET_ACQ_REL(char); +ATOMIC_CMPSET_ACQ_REL(short); + +ATOMIC_FCMPSET_ACQ_REL(char); +ATOMIC_FCMPSET_ACQ_REL(short); + +#undef __ATOMIC_REL +#undef __ATOMIC_ACQ #endif /* ! _MACHINE_ATOMIC_H_ */