Skip site navigation (1)Skip section navigation (2)
Date:      Mon, 1 Nov 2021 14:33:18 GMT
From:      Mark Johnston <markj@FreeBSD.org>
To:        src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org
Subject:   git: bf0986b742ab - stable/13 - Generalize bus_space(9) and atomic(9) sanitizer interceptors
Message-ID:  <202111011433.1A1EXICa021796@gitrepo.freebsd.org>

next in thread | raw e-mail | index | archive | help
The branch stable/13 has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=bf0986b742abaa493ed572391c798791468c702f

commit bf0986b742abaa493ed572391c798791468c702f
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2021-03-23 01:44:55 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2021-11-01 14:16:39 +0000

    Generalize bus_space(9) and atomic(9) sanitizer interceptors
    
    Make it easy to define interceptors for new sanitizer runtimes, rather
    than assuming KCSAN.  Lay a bit of groundwork for KASAN and KMSAN.
    
    When a sanitizer is compiled in, atomic(9) and bus_space(9) definitions
    in atomic_san.h are used by default instead of the inline
    implementations in the platform's atomic.h.  These definitions are
    implemented in the sanitizer runtime, which includes
    machine/{atomic,bus}.h with SAN_RUNTIME defined to pull in the actual
    implementations.
    
    No functional change intended.
    
    Sponsored by:   The FreeBSD Foundation
    
    (cherry picked from commit 3ead60236fd25ce64fece7ae4a453318ca18c119)
---
 sys/amd64/include/atomic.h    |  12 +-
 sys/arm64/arm64/bus_machdep.c |   4 +-
 sys/arm64/include/bus.h       |  10 +-
 sys/kern/kern_kcov.c          |   4 +-
 sys/kern/subr_coverage.c      |   4 +-
 sys/kern/subr_csan.c          |  10 +-
 sys/sys/atomic_san.h          | 630 ++++++++++++++++++++++--------------------
 sys/sys/bus_san.h             | 306 ++++++++++----------
 sys/x86/include/bus.h         |  18 +-
 sys/x86/x86/bus_machdep.c     |   4 +-
 10 files changed, 542 insertions(+), 460 deletions(-)

diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h
index 61cb79645c10..c821e77e5b8b 100644
--- a/sys/amd64/include/atomic.h
+++ b/sys/amd64/include/atomic.h
@@ -68,7 +68,15 @@
 #define	OFFSETOF_MONITORBUF	0x100
 #endif
 
-#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
+#ifndef SAN_RUNTIME
+#if defined(KASAN)
+#define	ATOMIC_SAN_PREFIX	kasan
+#elif defined(KCSAN)
+#define	ATOMIC_SAN_PREFIX	kcsan
+#endif
+#endif
+
+#ifdef ATOMIC_SAN_PREFIX
 #include <sys/atomic_san.h>
 #else
 #include <sys/atomic_common.h>
@@ -679,6 +687,6 @@ u_long	atomic_swap_long(volatile u_long *p, u_long v);
 
 #endif /* !WANT_FUNCTIONS */
 
-#endif /* KCSAN && !KCSAN_RUNTIME */
+#endif /* !ATOMIC_SAN_PREFIX */
 
 #endif /* !_MACHINE_ATOMIC_H_ */
diff --git a/sys/arm64/arm64/bus_machdep.c b/sys/arm64/arm64/bus_machdep.c
index 7fc83e6924f2..69d7c5b591b2 100644
--- a/sys/arm64/arm64/bus_machdep.c
+++ b/sys/arm64/arm64/bus_machdep.c
@@ -25,7 +25,9 @@
  *
  */
 
-#define	KCSAN_RUNTIME
+#if defined(KASAN) || defined(KCSAN)
+#define	SAN_RUNTIME
+#endif
 
 #include "opt_platform.h"
 
diff --git a/sys/arm64/include/bus.h b/sys/arm64/include/bus.h
index 5995aa0b4a79..a2bd432a5de5 100644
--- a/sys/arm64/include/bus.h
+++ b/sys/arm64/include/bus.h
@@ -91,7 +91,15 @@
 #define	BUS_SPACE_BARRIER_READ	0x01
 #define	BUS_SPACE_BARRIER_WRITE	0x02
 
-#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
+#ifndef SAN_RUNTIME
+#if defined(KASAN)
+#define	BUS_SAN_PREFIX	kasan
+#elif defined(KCSAN)
+#define	BUS_SAN_PREFIX	kcsan
+#endif
+#endif
+
+#ifdef BUS_SAN_PREFIX
 #include <sys/bus_san.h>
 #else
 
diff --git a/sys/kern/kern_kcov.c b/sys/kern/kern_kcov.c
index cedfa2c081ba..23e0da4cdb79 100644
--- a/sys/kern/kern_kcov.c
+++ b/sys/kern/kern_kcov.c
@@ -35,7 +35,9 @@
  * $FreeBSD$
  */
 
-#define	KCSAN_RUNTIME
+#ifdef KCSAN
+#define	SAN_RUNTIME
+#endif
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
diff --git a/sys/kern/subr_coverage.c b/sys/kern/subr_coverage.c
index f3ab27c2ca3d..9a719bcaecad 100644
--- a/sys/kern/subr_coverage.c
+++ b/sys/kern/subr_coverage.c
@@ -35,7 +35,9 @@
  * $FreeBSD$
  */
 
-#define	KCSAN_RUNTIME
+#ifdef KCSAN
+#define	SAN_RUNTIME
+#endif
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
diff --git a/sys/kern/subr_csan.c b/sys/kern/subr_csan.c
index ec2fd23729b2..06b0b6ebb020 100644
--- a/sys/kern/subr_csan.c
+++ b/sys/kern/subr_csan.c
@@ -30,7 +30,7 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
-#define	KCSAN_RUNTIME
+#define	SAN_RUNTIME
 
 #include "opt_ddb.h"
 
@@ -380,6 +380,7 @@ kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
 /* -------------------------------------------------------------------------- */
 
 #include <machine/atomic.h>
+#define	ATOMIC_SAN_PREFIX	kcsan
 #include <sys/atomic_san.h>
 
 #define	_CSAN_ATOMIC_FUNC_ADD(name, type)				\
@@ -684,10 +685,17 @@ CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel)
 CSAN_ATOMIC_FUNC_THREAD_FENCE(rel)
 CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst)
 
+void
+kcsan_atomic_interrupt_fence(void)
+{
+	atomic_interrupt_fence();
+}
+
 /* -------------------------------------------------------------------------- */
 
 #include <sys/bus.h>
 #include <machine/bus.h>
+#define	BUS_SAN_PREFIX		kcsan
 #include <sys/bus_san.h>
 
 int
diff --git a/sys/sys/atomic_san.h b/sys/sys/atomic_san.h
index c2b963ae8b92..5d10f58f7565 100644
--- a/sys/sys/atomic_san.h
+++ b/sys/sys/atomic_san.h
@@ -2,12 +2,16 @@
  * SPDX-License-Identifier: BSD-2-Clause
  *
  * Copyright (c) 2019 Andrew Turner
+ * Copyright (c) 2021 The FreeBSD Foundation
  *
  * This software was developed by SRI International and the University of
  * Cambridge Computer Laboratory (Department of Computer Science and
  * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
  * DARPA SSITH research programme.
  *
+ * Portions of this software were written by Mark Johnston under sponsorship
+ * by the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -39,342 +43,360 @@
 #error do not include this header, use machine/atomic.h
 #endif
 
-#define	KCSAN_ATOMIC_FUNC_1(op, name, type)				\
-	void kcsan_atomic_##op##_##name(volatile type *, type);		\
-	void kcsan_atomic_##op##_acq_##name(volatile type *, type);	\
-	void kcsan_atomic_##op##_rel_##name(volatile type *, type)
+#ifndef ATOMIC_SAN_PREFIX
+#error No sanitizer prefix available
+#endif
+
+#define	ATOMIC_SAN_FUNC_1(sp, op, name, type)				\
+	void sp##_atomic_##op##_##name(volatile type *, type);		\
+	void sp##_atomic_##op##_acq_##name(volatile type *, type);	\
+	void sp##_atomic_##op##_rel_##name(volatile type *, type)
 
-#define	KCSAN_ATOMIC_CMPSET(name, type)					\
-	int kcsan_atomic_cmpset_##name(volatile type *, type, type);	\
-	int kcsan_atomic_cmpset_acq_##name(volatile type *, type, type); \
-	int kcsan_atomic_cmpset_rel_##name(volatile type *, type, type)
+#define	ATOMIC_SAN_CMPSET(sp, name, type)				\
+	int sp##_atomic_cmpset_##name(volatile type *, type, type);	\
+	int sp##_atomic_cmpset_acq_##name(volatile type *, type, type); \
+	int sp##_atomic_cmpset_rel_##name(volatile type *, type, type)
 
-#define	KCSAN_ATOMIC_FCMPSET(name, type)				\
-	int kcsan_atomic_fcmpset_##name(volatile type *, type *, type);	\
-	int kcsan_atomic_fcmpset_acq_##name(volatile type *, type *, type); \
-	int kcsan_atomic_fcmpset_rel_##name(volatile type *, type *, type)
+#define	ATOMIC_SAN_FCMPSET(sp, name, type)				\
+	int sp##_atomic_fcmpset_##name(volatile type *, type *, type);	\
+	int sp##_atomic_fcmpset_acq_##name(volatile type *, type *, type); \
+	int sp##_atomic_fcmpset_rel_##name(volatile type *, type *, type)
 
-#define	KCSAN_ATOMIC_READ(op, name, type)				\
-	type kcsan_atomic_##op##_##name(volatile type *, type)
+#define	ATOMIC_SAN_READ(sp, op, name, type)				\
+	type sp##_atomic_##op##_##name(volatile type *, type)
 
-#define	KCSAN_ATOMIC_READANDCLEAR(name, type)				\
-	type kcsan_atomic_readandclear_##name(volatile type *)
+#define	ATOMIC_SAN_READANDCLEAR(sp, name, type)				\
+	type sp##_atomic_readandclear_##name(volatile type *)
 
-#define	KCSAN_ATOMIC_LOAD(name, type)					\
-	type kcsan_atomic_load_##name(volatile type *);			\
-	type kcsan_atomic_load_acq_##name(volatile type *)
+#define	ATOMIC_SAN_LOAD(sp, name, type)					\
+	type sp##_atomic_load_##name(volatile type *);			\
+	type sp##_atomic_load_acq_##name(volatile type *)
 
-#define	KCSAN_ATOMIC_STORE(name, type)					\
-	void kcsan_atomic_store_##name(volatile type *, type);		\
-	void kcsan_atomic_store_rel_##name(volatile type *, type)
+#define	ATOMIC_SAN_STORE(sp, name, type)				\
+	void sp##_atomic_store_##name(volatile type *, type);		\
+	void sp##_atomic_store_rel_##name(volatile type *, type)
 
-#define	KCSAN_ATOMIC_TEST(op, name, type)				\
-	int kcsan_atomic_##op##_##name(volatile type *, u_int);		\
-	int kcsan_atomic_##op##_acq_##name(volatile type *, u_int)
+#define	ATOMIC_SAN_TEST(sp, op, name, type)				\
+	int sp##_atomic_##op##_##name(volatile type *, u_int);		\
+	int sp##_atomic_##op##_acq_##name(volatile type *, u_int)
 
-#define	KCSAN_ATOMIC_FUNCS(name, type)					\
-	KCSAN_ATOMIC_FUNC_1(add, name, type);				\
-	KCSAN_ATOMIC_FUNC_1(clear, name, type);				\
-	KCSAN_ATOMIC_CMPSET(name, type);				\
-	KCSAN_ATOMIC_FCMPSET(name, type);				\
-	KCSAN_ATOMIC_READ(fetchadd, name, type);			\
-	KCSAN_ATOMIC_LOAD(name, type);					\
-	KCSAN_ATOMIC_READANDCLEAR(name, type);				\
-	KCSAN_ATOMIC_FUNC_1(set, name, type);				\
-	KCSAN_ATOMIC_FUNC_1(subtract, name, type);			\
-	KCSAN_ATOMIC_STORE(name, type);					\
-	KCSAN_ATOMIC_READ(swap, name, type);				\
-	KCSAN_ATOMIC_TEST(testandclear, name, type);			\
-	KCSAN_ATOMIC_TEST(testandset, name, type)
+#define	ATOMIC_SAN_THREAD_FENCE(sp)					\
+	void sp##_atomic_thread_fence_acq(void);			\
+	void sp##_atomic_thread_fence_rel(void);			\
+	void sp##_atomic_thread_fence_acq_rel(void);			\
+	void sp##_atomic_thread_fence_seq_cst(void);			\
+	void sp##_atomic_interrupt_fence(void)
 
-KCSAN_ATOMIC_FUNCS(char, uint8_t);
-KCSAN_ATOMIC_FUNCS(short, uint16_t);
-KCSAN_ATOMIC_FUNCS(int, u_int);
-KCSAN_ATOMIC_FUNCS(long, u_long);
-KCSAN_ATOMIC_FUNCS(ptr, uintptr_t);
-KCSAN_ATOMIC_FUNCS(8, uint8_t);
-KCSAN_ATOMIC_FUNCS(16, uint16_t);
-KCSAN_ATOMIC_FUNCS(32, uint32_t);
-KCSAN_ATOMIC_FUNCS(64, uint64_t);
+#define	_ATOMIC_SAN_FUNCS(sp, name, type)				\
+	ATOMIC_SAN_FUNC_1(sp, add, name, type);				\
+	ATOMIC_SAN_FUNC_1(sp, clear, name, type);			\
+	ATOMIC_SAN_CMPSET(sp, name, type);				\
+	ATOMIC_SAN_FCMPSET(sp, name, type);				\
+	ATOMIC_SAN_READ(sp, fetchadd, name, type);			\
+	ATOMIC_SAN_LOAD(sp, name, type);				\
+	ATOMIC_SAN_READANDCLEAR(sp, name, type);			\
+	ATOMIC_SAN_FUNC_1(sp, set, name, type);				\
+	ATOMIC_SAN_FUNC_1(sp, subtract, name, type);			\
+	ATOMIC_SAN_STORE(sp, name, type);				\
+	ATOMIC_SAN_READ(sp, swap, name, type);				\
+	ATOMIC_SAN_TEST(sp, testandclear, name, type);			\
+	ATOMIC_SAN_TEST(sp, testandset, name, type);			\
+	ATOMIC_SAN_THREAD_FENCE(sp);
 
-void	kcsan_atomic_thread_fence_acq(void);
-void	kcsan_atomic_thread_fence_acq_rel(void);
-void	kcsan_atomic_thread_fence_rel(void);
-void	kcsan_atomic_thread_fence_seq_cst(void);
+#define	ATOMIC_SAN_FUNCS(name, type)					\
+	_ATOMIC_SAN_FUNCS(ATOMIC_SAN_PREFIX, name, type)
 
-#ifndef KCSAN_RUNTIME
+ATOMIC_SAN_FUNCS(char, uint8_t);
+ATOMIC_SAN_FUNCS(short, uint16_t);
+ATOMIC_SAN_FUNCS(int, u_int);
+ATOMIC_SAN_FUNCS(long, u_long);
+ATOMIC_SAN_FUNCS(ptr, uintptr_t);
+ATOMIC_SAN_FUNCS(8, uint8_t);
+ATOMIC_SAN_FUNCS(16, uint16_t);
+ATOMIC_SAN_FUNCS(32, uint32_t);
+ATOMIC_SAN_FUNCS(64, uint64_t);
+
+#ifndef SAN_RUNTIME
+
+/*
+ * Redirect uses of an atomic(9) function to the sanitizer's interceptor.
+ * For instance, KASAN callers of atomic_add_char() will be redirected to
+ * kasan_atomic_add_char().
+ */
+#define	ATOMIC_SAN(func)		\
+	__CONCAT(ATOMIC_SAN_PREFIX, __CONCAT(_atomic_, func))
 
-#define	atomic_add_char			kcsan_atomic_add_char
-#define	atomic_add_acq_char		kcsan_atomic_add_acq_char
-#define	atomic_add_rel_char		kcsan_atomic_add_rel_char
-#define	atomic_clear_char		kcsan_atomic_clear_char
-#define	atomic_clear_acq_char		kcsan_atomic_clear_acq_char
-#define	atomic_clear_rel_char		kcsan_atomic_clear_rel_char
-#define	atomic_cmpset_char		kcsan_atomic_cmpset_char
-#define	atomic_cmpset_acq_char		kcsan_atomic_cmpset_acq_char
-#define	atomic_cmpset_rel_char		kcsan_atomic_cmpset_rel_char
-#define	atomic_fcmpset_char		kcsan_atomic_fcmpset_char
-#define	atomic_fcmpset_acq_char		kcsan_atomic_fcmpset_acq_char
-#define	atomic_fcmpset_rel_char		kcsan_atomic_fcmpset_rel_char
-#define	atomic_fetchadd_char		kcsan_atomic_fetchadd_char
-#define	atomic_load_char		kcsan_atomic_load_char
-#define	atomic_load_acq_char		kcsan_atomic_load_acq_char
-#define	atomic_readandclear_char	kcsan_atomic_readandclear_char
-#define	atomic_set_char			kcsan_atomic_set_char
-#define	atomic_set_acq_char		kcsan_atomic_set_acq_char
-#define	atomic_set_rel_char		kcsan_atomic_set_rel_char
-#define	atomic_subtract_char		kcsan_atomic_subtract_char
-#define	atomic_subtract_acq_char	kcsan_atomic_subtract_acq_char
-#define	atomic_subtract_rel_char	kcsan_atomic_subtract_rel_char
-#define	atomic_store_char		kcsan_atomic_store_char
-#define	atomic_store_rel_char		kcsan_atomic_store_rel_char
-#define	atomic_swap_char		kcsan_atomic_swap_char
-#define	atomic_testandclear_char	kcsan_atomic_testandclear_char
-#define	atomic_testandset_char		kcsan_atomic_testandset_char
+#define	atomic_add_char			ATOMIC_SAN(add_char)
+#define	atomic_add_acq_char		ATOMIC_SAN(add_acq_char)
+#define	atomic_add_rel_char		ATOMIC_SAN(add_rel_char)
+#define	atomic_clear_char		ATOMIC_SAN(clear_char)
+#define	atomic_clear_acq_char		ATOMIC_SAN(clear_acq_char)
+#define	atomic_clear_rel_char		ATOMIC_SAN(clear_rel_char)
+#define	atomic_cmpset_char		ATOMIC_SAN(cmpset_char)
+#define	atomic_cmpset_acq_char		ATOMIC_SAN(cmpset_acq_char)
+#define	atomic_cmpset_rel_char		ATOMIC_SAN(cmpset_rel_char)
+#define	atomic_fcmpset_char		ATOMIC_SAN(fcmpset_char)
+#define	atomic_fcmpset_acq_char		ATOMIC_SAN(fcmpset_acq_char)
+#define	atomic_fcmpset_rel_char		ATOMIC_SAN(fcmpset_rel_char)
+#define	atomic_fetchadd_char		ATOMIC_SAN(fetchadd_char)
+#define	atomic_load_char		ATOMIC_SAN(load_char)
+#define	atomic_load_acq_char		ATOMIC_SAN(load_acq_char)
+#define	atomic_readandclear_char	ATOMIC_SAN(readandclear_char)
+#define	atomic_set_char			ATOMIC_SAN(set_char)
+#define	atomic_set_acq_char		ATOMIC_SAN(set_acq_char)
+#define	atomic_set_rel_char		ATOMIC_SAN(set_rel_char)
+#define	atomic_subtract_char		ATOMIC_SAN(subtract_char)
+#define	atomic_subtract_acq_char	ATOMIC_SAN(subtract_acq_char)
+#define	atomic_subtract_rel_char	ATOMIC_SAN(subtract_rel_char)
+#define	atomic_store_char		ATOMIC_SAN(store_char)
+#define	atomic_store_rel_char		ATOMIC_SAN(store_rel_char)
+#define	atomic_swap_char		ATOMIC_SAN(swap_char)
+#define	atomic_testandclear_char	ATOMIC_SAN(testandclear_char)
+#define	atomic_testandset_char		ATOMIC_SAN(testandset_char)
 
-#define	atomic_add_short		kcsan_atomic_add_short
-#define	atomic_add_acq_short		kcsan_atomic_add_acq_short
-#define	atomic_add_rel_short		kcsan_atomic_add_rel_short
-#define	atomic_clear_short		kcsan_atomic_clear_short
-#define	atomic_clear_acq_short		kcsan_atomic_clear_acq_short
-#define	atomic_clear_rel_short		kcsan_atomic_clear_rel_short
-#define	atomic_cmpset_short		kcsan_atomic_cmpset_short
-#define	atomic_cmpset_acq_short		kcsan_atomic_cmpset_acq_short
-#define	atomic_cmpset_rel_short		kcsan_atomic_cmpset_rel_short
-#define	atomic_fcmpset_short		kcsan_atomic_fcmpset_short
-#define	atomic_fcmpset_acq_short	kcsan_atomic_fcmpset_acq_short
-#define	atomic_fcmpset_rel_short	kcsan_atomic_fcmpset_rel_short
-#define	atomic_fetchadd_short		kcsan_atomic_fetchadd_short
-#define	atomic_load_short		kcsan_atomic_load_short
-#define	atomic_load_acq_short		kcsan_atomic_load_acq_short
-#define	atomic_readandclear_short	kcsan_atomic_readandclear_short
-#define	atomic_set_short		kcsan_atomic_set_short
-#define	atomic_set_acq_short		kcsan_atomic_set_acq_short
-#define	atomic_set_rel_short		kcsan_atomic_set_rel_short
-#define	atomic_subtract_short		kcsan_atomic_subtract_short
-#define	atomic_subtract_acq_short	kcsan_atomic_subtract_acq_short
-#define	atomic_subtract_rel_short	kcsan_atomic_subtract_rel_short
-#define	atomic_store_short		kcsan_atomic_store_short
-#define	atomic_store_rel_short		kcsan_atomic_store_rel_short
-#define	atomic_swap_short		kcsan_atomic_swap_short
-#define	atomic_testandclear_short	kcsan_atomic_testandclear_short
-#define	atomic_testandset_short		kcsan_atomic_testandset_short
+#define	atomic_add_short		ATOMIC_SAN(add_short)
+#define	atomic_add_acq_short		ATOMIC_SAN(add_acq_short)
+#define	atomic_add_rel_short		ATOMIC_SAN(add_rel_short)
+#define	atomic_clear_short		ATOMIC_SAN(clear_short)
+#define	atomic_clear_acq_short		ATOMIC_SAN(clear_acq_short)
+#define	atomic_clear_rel_short		ATOMIC_SAN(clear_rel_short)
+#define	atomic_cmpset_short		ATOMIC_SAN(cmpset_short)
+#define	atomic_cmpset_acq_short		ATOMIC_SAN(cmpset_acq_short)
+#define	atomic_cmpset_rel_short		ATOMIC_SAN(cmpset_rel_short)
+#define	atomic_fcmpset_short		ATOMIC_SAN(fcmpset_short)
+#define	atomic_fcmpset_acq_short	ATOMIC_SAN(fcmpset_acq_short)
+#define	atomic_fcmpset_rel_short	ATOMIC_SAN(fcmpset_rel_short)
+#define	atomic_fetchadd_short		ATOMIC_SAN(fetchadd_short)
+#define	atomic_load_short		ATOMIC_SAN(load_short)
+#define	atomic_load_acq_short		ATOMIC_SAN(load_acq_short)
+#define	atomic_readandclear_short	ATOMIC_SAN(readandclear_short)
+#define	atomic_set_short		ATOMIC_SAN(set_short)
+#define	atomic_set_acq_short		ATOMIC_SAN(set_acq_short)
+#define	atomic_set_rel_short		ATOMIC_SAN(set_rel_short)
+#define	atomic_subtract_short		ATOMIC_SAN(subtract_short)
+#define	atomic_subtract_acq_short	ATOMIC_SAN(subtract_acq_short)
+#define	atomic_subtract_rel_short	ATOMIC_SAN(subtract_rel_short)
+#define	atomic_store_short		ATOMIC_SAN(store_short)
+#define	atomic_store_rel_short		ATOMIC_SAN(store_rel_short)
+#define	atomic_swap_short		ATOMIC_SAN(swap_short)
+#define	atomic_testandclear_short	ATOMIC_SAN(testandclear_short)
+#define	atomic_testandset_short		ATOMIC_SAN(testandset_short)
 
-#define	atomic_add_int			kcsan_atomic_add_int
-#define	atomic_add_acq_int		kcsan_atomic_add_acq_int
-#define	atomic_add_rel_int		kcsan_atomic_add_rel_int
-#define	atomic_clear_int		kcsan_atomic_clear_int
-#define	atomic_clear_acq_int		kcsan_atomic_clear_acq_int
-#define	atomic_clear_rel_int		kcsan_atomic_clear_rel_int
-#define	atomic_cmpset_int		kcsan_atomic_cmpset_int
-#define	atomic_cmpset_acq_int		kcsan_atomic_cmpset_acq_int
-#define	atomic_cmpset_rel_int		kcsan_atomic_cmpset_rel_int
-#define	atomic_fcmpset_int		kcsan_atomic_fcmpset_int
-#define	atomic_fcmpset_acq_int		kcsan_atomic_fcmpset_acq_int
-#define	atomic_fcmpset_rel_int		kcsan_atomic_fcmpset_rel_int
-#define	atomic_fetchadd_int		kcsan_atomic_fetchadd_int
-#define	atomic_load_int			kcsan_atomic_load_int
-#define	atomic_load_acq_int		kcsan_atomic_load_acq_int
-#define	atomic_readandclear_int		kcsan_atomic_readandclear_int
-#define	atomic_set_int			kcsan_atomic_set_int
-#define	atomic_set_acq_int		kcsan_atomic_set_acq_int
-#define	atomic_set_rel_int		kcsan_atomic_set_rel_int
-#define	atomic_subtract_int		kcsan_atomic_subtract_int
-#define	atomic_subtract_acq_int		kcsan_atomic_subtract_acq_int
-#define	atomic_subtract_rel_int		kcsan_atomic_subtract_rel_int
-#define	atomic_store_int		kcsan_atomic_store_int
-#define	atomic_store_rel_int		kcsan_atomic_store_rel_int
-#define	atomic_swap_int			kcsan_atomic_swap_int
-#define	atomic_testandclear_int		kcsan_atomic_testandclear_int
-#define	atomic_testandset_int		kcsan_atomic_testandset_int
+#define	atomic_add_int			ATOMIC_SAN(add_int)
+#define	atomic_add_acq_int		ATOMIC_SAN(add_acq_int)
+#define	atomic_add_rel_int		ATOMIC_SAN(add_rel_int)
+#define	atomic_clear_int		ATOMIC_SAN(clear_int)
+#define	atomic_clear_acq_int		ATOMIC_SAN(clear_acq_int)
+#define	atomic_clear_rel_int		ATOMIC_SAN(clear_rel_int)
+#define	atomic_cmpset_int		ATOMIC_SAN(cmpset_int)
+#define	atomic_cmpset_acq_int		ATOMIC_SAN(cmpset_acq_int)
+#define	atomic_cmpset_rel_int		ATOMIC_SAN(cmpset_rel_int)
+#define	atomic_fcmpset_int		ATOMIC_SAN(fcmpset_int)
+#define	atomic_fcmpset_acq_int		ATOMIC_SAN(fcmpset_acq_int)
+#define	atomic_fcmpset_rel_int		ATOMIC_SAN(fcmpset_rel_int)
+#define	atomic_fetchadd_int		ATOMIC_SAN(fetchadd_int)
+#define	atomic_load_int			ATOMIC_SAN(load_int)
+#define	atomic_load_acq_int		ATOMIC_SAN(load_acq_int)
+#define	atomic_readandclear_int		ATOMIC_SAN(readandclear_int)
+#define	atomic_set_int			ATOMIC_SAN(set_int)
+#define	atomic_set_acq_int		ATOMIC_SAN(set_acq_int)
+#define	atomic_set_rel_int		ATOMIC_SAN(set_rel_int)
+#define	atomic_subtract_int		ATOMIC_SAN(subtract_int)
+#define	atomic_subtract_acq_int		ATOMIC_SAN(subtract_acq_int)
+#define	atomic_subtract_rel_int		ATOMIC_SAN(subtract_rel_int)
+#define	atomic_store_int		ATOMIC_SAN(store_int)
+#define	atomic_store_rel_int		ATOMIC_SAN(store_rel_int)
+#define	atomic_swap_int			ATOMIC_SAN(swap_int)
+#define	atomic_testandclear_int		ATOMIC_SAN(testandclear_int)
+#define	atomic_testandset_int		ATOMIC_SAN(testandset_int)
 
-#define	atomic_add_long			kcsan_atomic_add_long
-#define	atomic_add_acq_long		kcsan_atomic_add_acq_long
-#define	atomic_add_rel_long		kcsan_atomic_add_rel_long
-#define	atomic_clear_long		kcsan_atomic_clear_long
-#define	atomic_clear_acq_long		kcsan_atomic_clear_acq_long
-#define	atomic_clear_rel_long		kcsan_atomic_clear_rel_long
-#define	atomic_cmpset_long		kcsan_atomic_cmpset_long
-#define	atomic_cmpset_acq_long		kcsan_atomic_cmpset_acq_long
-#define	atomic_cmpset_rel_long		kcsan_atomic_cmpset_rel_long
-#define	atomic_fcmpset_long		kcsan_atomic_fcmpset_long
-#define	atomic_fcmpset_acq_long		kcsan_atomic_fcmpset_acq_long
-#define	atomic_fcmpset_rel_long		kcsan_atomic_fcmpset_rel_long
-#define	atomic_fetchadd_long		kcsan_atomic_fetchadd_long
-#define	atomic_load_long		kcsan_atomic_load_long
-#define	atomic_load_acq_long		kcsan_atomic_load_acq_long
-#define	atomic_readandclear_long	kcsan_atomic_readandclear_long
-#define	atomic_set_long			kcsan_atomic_set_long
-#define	atomic_set_acq_long		kcsan_atomic_set_acq_long
-#define	atomic_set_rel_long		kcsan_atomic_set_rel_long
-#define	atomic_subtract_long		kcsan_atomic_subtract_long
-#define	atomic_subtract_acq_long	kcsan_atomic_subtract_acq_long
-#define	atomic_subtract_rel_long	kcsan_atomic_subtract_rel_long
-#define	atomic_store_long		kcsan_atomic_store_long
-#define	atomic_store_rel_long		kcsan_atomic_store_rel_long
-#define	atomic_swap_long		kcsan_atomic_swap_long
-#define	atomic_testandclear_long	kcsan_atomic_testandclear_long
-#define	atomic_testandset_long		kcsan_atomic_testandset_long
-#define	atomic_testandset_acq_long	kcsan_atomic_testandset_acq_long
+#define	atomic_add_long			ATOMIC_SAN(add_long)
+#define	atomic_add_acq_long		ATOMIC_SAN(add_acq_long)
+#define	atomic_add_rel_long		ATOMIC_SAN(add_rel_long)
+#define	atomic_clear_long		ATOMIC_SAN(clear_long)
+#define	atomic_clear_acq_long		ATOMIC_SAN(clear_acq_long)
+#define	atomic_clear_rel_long		ATOMIC_SAN(clear_rel_long)
+#define	atomic_cmpset_long		ATOMIC_SAN(cmpset_long)
+#define	atomic_cmpset_acq_long		ATOMIC_SAN(cmpset_acq_long)
+#define	atomic_cmpset_rel_long		ATOMIC_SAN(cmpset_rel_long)
+#define	atomic_fcmpset_long		ATOMIC_SAN(fcmpset_long)
+#define	atomic_fcmpset_acq_long		ATOMIC_SAN(fcmpset_acq_long)
+#define	atomic_fcmpset_rel_long		ATOMIC_SAN(fcmpset_rel_long)
+#define	atomic_fetchadd_long		ATOMIC_SAN(fetchadd_long)
+#define	atomic_load_long		ATOMIC_SAN(load_long)
+#define	atomic_load_acq_long		ATOMIC_SAN(load_acq_long)
+#define	atomic_readandclear_long	ATOMIC_SAN(readandclear_long)
+#define	atomic_set_long			ATOMIC_SAN(set_long)
+#define	atomic_set_acq_long		ATOMIC_SAN(set_acq_long)
+#define	atomic_set_rel_long		ATOMIC_SAN(set_rel_long)
+#define	atomic_subtract_long		ATOMIC_SAN(subtract_long)
+#define	atomic_subtract_acq_long	ATOMIC_SAN(subtract_acq_long)
+#define	atomic_subtract_rel_long	ATOMIC_SAN(subtract_rel_long)
+#define	atomic_store_long		ATOMIC_SAN(store_long)
+#define	atomic_store_rel_long		ATOMIC_SAN(store_rel_long)
+#define	atomic_swap_long		ATOMIC_SAN(swap_long)
+#define	atomic_testandclear_long	ATOMIC_SAN(testandclear_long)
+#define	atomic_testandset_long		ATOMIC_SAN(testandset_long)
+#define	atomic_testandset_acq_long	ATOMIC_SAN(testandset_acq_long)
 
-#define	atomic_add_ptr			kcsan_atomic_add_ptr
-#define	atomic_add_acq_ptr		kcsan_atomic_add_acq_ptr
-#define	atomic_add_rel_ptr		kcsan_atomic_add_rel_ptr
-#define	atomic_clear_ptr		kcsan_atomic_clear_ptr
-#define	atomic_clear_acq_ptr		kcsan_atomic_clear_acq_ptr
-#define	atomic_clear_rel_ptr		kcsan_atomic_clear_rel_ptr
-#define	atomic_cmpset_ptr		kcsan_atomic_cmpset_ptr
-#define	atomic_cmpset_acq_ptr		kcsan_atomic_cmpset_acq_ptr
-#define	atomic_cmpset_rel_ptr		kcsan_atomic_cmpset_rel_ptr
-#define	atomic_fcmpset_ptr		kcsan_atomic_fcmpset_ptr
-#define	atomic_fcmpset_acq_ptr		kcsan_atomic_fcmpset_acq_ptr
-#define	atomic_fcmpset_rel_ptr		kcsan_atomic_fcmpset_rel_ptr
-#define	atomic_fetchadd_ptr		kcsan_atomic_fetchadd_ptr
+#define	atomic_add_ptr			ATOMIC_SAN(add_ptr)
+#define	atomic_add_acq_ptr		ATOMIC_SAN(add_acq_ptr)
+#define	atomic_add_rel_ptr		ATOMIC_SAN(add_rel_ptr)
+#define	atomic_clear_ptr		ATOMIC_SAN(clear_ptr)
+#define	atomic_clear_acq_ptr		ATOMIC_SAN(clear_acq_ptr)
+#define	atomic_clear_rel_ptr		ATOMIC_SAN(clear_rel_ptr)
+#define	atomic_cmpset_ptr		ATOMIC_SAN(cmpset_ptr)
+#define	atomic_cmpset_acq_ptr		ATOMIC_SAN(cmpset_acq_ptr)
+#define	atomic_cmpset_rel_ptr		ATOMIC_SAN(cmpset_rel_ptr)
+#define	atomic_fcmpset_ptr		ATOMIC_SAN(fcmpset_ptr)
+#define	atomic_fcmpset_acq_ptr		ATOMIC_SAN(fcmpset_acq_ptr)
+#define	atomic_fcmpset_rel_ptr		ATOMIC_SAN(fcmpset_rel_ptr)
+#define	atomic_fetchadd_ptr		ATOMIC_SAN(fetchadd_ptr)
 #define	atomic_load_ptr(x)		({					\
 	__typeof(*x) __retptr;							\
-	__retptr = (void *)kcsan_atomic_load_ptr((volatile uintptr_t *)(x));	\
+	__retptr = (void *)ATOMIC_SAN(load_ptr)((volatile uintptr_t *)(x));	\
 	__retptr;								\
 })
-#define	atomic_load_acq_ptr		kcsan_atomic_load_acq_ptr
+#define	atomic_load_acq_ptr		ATOMIC_SAN(load_acq_ptr)
 #define	atomic_load_consume_ptr(x)	({					\
 	__typeof(*x) __retptr;							\
-	__retptr = (void *)kcsan_atomic_load_acq_ptr((volatile uintptr_t *)(x));\
+	__retptr = (void *)atomic_load_acq_ptr((volatile uintptr_t *)(x));\
 	__retptr;								\
 })
-#define	atomic_readandclear_ptr		kcsan_atomic_readandclear_ptr
-#define	atomic_set_ptr			kcsan_atomic_set_ptr
-#define	atomic_set_acq_ptr		kcsan_atomic_set_acq_ptr
-#define	atomic_set_rel_ptr		kcsan_atomic_set_rel_ptr
-#define	atomic_subtract_ptr		kcsan_atomic_subtract_ptr
-#define	atomic_subtract_acq_ptr		kcsan_atomic_subtract_acq_ptr
-#define	atomic_subtract_rel_ptr		kcsan_atomic_subtract_rel_ptr
+#define	atomic_readandclear_ptr		ATOMIC_SAN(readandclear_ptr)
+#define	atomic_set_ptr			ATOMIC_SAN(set_ptr)
+#define	atomic_set_acq_ptr		ATOMIC_SAN(set_acq_ptr)
+#define	atomic_set_rel_ptr		ATOMIC_SAN(set_rel_ptr)
+#define	atomic_subtract_ptr		ATOMIC_SAN(subtract_ptr)
+#define	atomic_subtract_acq_ptr		ATOMIC_SAN(subtract_acq_ptr)
+#define	atomic_subtract_rel_ptr		ATOMIC_SAN(subtract_rel_ptr)
 #define	atomic_store_ptr(x, v)		({					\
 	__typeof(*x) __value = (v);						\
-	kcsan_atomic_store_ptr((volatile uintptr_t *)(x), (uintptr_t)(__value));\
+	ATOMIC_SAN(store_ptr)((volatile uintptr_t *)(x), (uintptr_t)(__value));\
 })
-#define	atomic_store_rel_ptr		kcsan_atomic_store_rel_ptr
-#define	atomic_swap_ptr			kcsan_atomic_swap_ptr
-#define	atomic_testandclear_ptr		kcsan_atomic_testandclear_ptr
-#define	atomic_testandset_ptr		kcsan_atomic_testandset_ptr
+#define	atomic_store_rel_ptr		ATOMIC_SAN(store_rel_ptr)
+#define	atomic_swap_ptr			ATOMIC_SAN(swap_ptr)
+#define	atomic_testandclear_ptr		ATOMIC_SAN(testandclear_ptr)
+#define	atomic_testandset_ptr		ATOMIC_SAN(testandset_ptr)
 
-#define	atomic_add_8			kcsan_atomic_add_8
-#define	atomic_add_acq_8		kcsan_atomic_add_acq_8
-#define	atomic_add_rel_8		kcsan_atomic_add_rel_8
-#define	atomic_clear_8			kcsan_atomic_clear_8
-#define	atomic_clear_acq_8		kcsan_atomic_clear_acq_8
-#define	atomic_clear_rel_8		kcsan_atomic_clear_rel_8
-#define	atomic_cmpset_8			kcsan_atomic_cmpset_8
-#define	atomic_cmpset_acq_8		kcsan_atomic_cmpset_acq_8
-#define	atomic_cmpset_rel_8		kcsan_atomic_cmpset_rel_8
-#define	atomic_fcmpset_8		kcsan_atomic_fcmpset_8
-#define	atomic_fcmpset_acq_8		kcsan_atomic_fcmpset_acq_8
-#define	atomic_fcmpset_rel_8		kcsan_atomic_fcmpset_rel_8
-#define	atomic_fetchadd_8		kcsan_atomic_fetchadd_8
-#define	atomic_load_8			kcsan_atomic_load_8
-#define	atomic_load_acq_8		kcsan_atomic_load_acq_8
-#define	atomic_readandclear_8		kcsan_atomic_readandclear_8
-#define	atomic_set_8			kcsan_atomic_set_8
-#define	atomic_set_acq_8		kcsan_atomic_set_acq_8
-#define	atomic_set_rel_8		kcsan_atomic_set_rel_8
-#define	atomic_subtract_8		kcsan_atomic_subtract_8
-#define	atomic_subtract_acq_8		kcsan_atomic_subtract_acq_8
-#define	atomic_subtract_rel_8		kcsan_atomic_subtract_rel_8
-#define	atomic_store_8			kcsan_atomic_store_8
-#define	atomic_store_rel_8		kcsan_atomic_store_rel_8
-#define	atomic_swap_8			kcsan_atomic_swap_8
-#define	atomic_testandclear_8		kcsan_atomic_testandclear_8
-#define	atomic_testandset_8		kcsan_atomic_testandset_8
+#define	atomic_add_8			ATOMIC_SAN(add_8)
+#define	atomic_add_acq_8		ATOMIC_SAN(add_acq_8)
+#define	atomic_add_rel_8		ATOMIC_SAN(add_rel_8)
+#define	atomic_clear_8			ATOMIC_SAN(clear_8)
+#define	atomic_clear_acq_8		ATOMIC_SAN(clear_acq_8)
+#define	atomic_clear_rel_8		ATOMIC_SAN(clear_rel_8)
+#define	atomic_cmpset_8			ATOMIC_SAN(cmpset_8)
+#define	atomic_cmpset_acq_8		ATOMIC_SAN(cmpset_acq_8)
+#define	atomic_cmpset_rel_8		ATOMIC_SAN(cmpset_rel_8)
+#define	atomic_fcmpset_8		ATOMIC_SAN(fcmpset_8)
+#define	atomic_fcmpset_acq_8		ATOMIC_SAN(fcmpset_acq_8)
+#define	atomic_fcmpset_rel_8		ATOMIC_SAN(fcmpset_rel_8)
+#define	atomic_fetchadd_8		ATOMIC_SAN(fetchadd_8)
+#define	atomic_load_8			ATOMIC_SAN(load_8)
+#define	atomic_load_acq_8		ATOMIC_SAN(load_acq_8)
+#define	atomic_readandclear_8		ATOMIC_SAN(readandclear_8)
+#define	atomic_set_8			ATOMIC_SAN(set_8)
+#define	atomic_set_acq_8		ATOMIC_SAN(set_acq_8)
+#define	atomic_set_rel_8		ATOMIC_SAN(set_rel_8)
+#define	atomic_subtract_8		ATOMIC_SAN(subtract_8)
+#define	atomic_subtract_acq_8		ATOMIC_SAN(subtract_acq_8)
+#define	atomic_subtract_rel_8		ATOMIC_SAN(subtract_rel_8)
+#define	atomic_store_8			ATOMIC_SAN(store_8)
+#define	atomic_store_rel_8		ATOMIC_SAN(store_rel_8)
+#define	atomic_swap_8			ATOMIC_SAN(swap_8)
+#define	atomic_testandclear_8		ATOMIC_SAN(testandclear_8)
+#define	atomic_testandset_8		ATOMIC_SAN(testandset_8)
 
-#define	atomic_add_16			kcsan_atomic_add_16
-#define	atomic_add_acq_16		kcsan_atomic_add_acq_16
-#define	atomic_add_rel_16		kcsan_atomic_add_rel_16
-#define	atomic_clear_16			kcsan_atomic_clear_16
-#define	atomic_clear_acq_16		kcsan_atomic_clear_acq_16
-#define	atomic_clear_rel_16		kcsan_atomic_clear_rel_16
-#define	atomic_cmpset_16		kcsan_atomic_cmpset_16
-#define	atomic_cmpset_acq_16		kcsan_atomic_cmpset_acq_16
-#define	atomic_cmpset_rel_16		kcsan_atomic_cmpset_rel_16
-#define	atomic_fcmpset_16		kcsan_atomic_fcmpset_16
-#define	atomic_fcmpset_acq_16		kcsan_atomic_fcmpset_acq_16
-#define	atomic_fcmpset_rel_16		kcsan_atomic_fcmpset_rel_16
-#define	atomic_fetchadd_16		kcsan_atomic_fetchadd_16
-#define	atomic_load_16			kcsan_atomic_load_16
-#define	atomic_load_acq_16		kcsan_atomic_load_acq_16
-#define	atomic_readandclear_16		kcsan_atomic_readandclear_16
-#define	atomic_set_16			kcsan_atomic_set_16
-#define	atomic_set_acq_16		kcsan_atomic_set_acq_16
-#define	atomic_set_rel_16		kcsan_atomic_set_rel_16
-#define	atomic_subtract_16		kcsan_atomic_subtract_16
-#define	atomic_subtract_acq_16		kcsan_atomic_subtract_acq_16
-#define	atomic_subtract_rel_16		kcsan_atomic_subtract_rel_16
-#define	atomic_store_16			kcsan_atomic_store_16
-#define	atomic_store_rel_16		kcsan_atomic_store_rel_16
-#define	atomic_swap_16			kcsan_atomic_swap_16
-#define	atomic_testandclear_16		kcsan_atomic_testandclear_16
-#define	atomic_testandset_16		kcsan_atomic_testandset_16
+#define	atomic_add_16			ATOMIC_SAN(add_16)
+#define	atomic_add_acq_16		ATOMIC_SAN(add_acq_16)
+#define	atomic_add_rel_16		ATOMIC_SAN(add_rel_16)
+#define	atomic_clear_16			ATOMIC_SAN(clear_16)
+#define	atomic_clear_acq_16		ATOMIC_SAN(clear_acq_16)
+#define	atomic_clear_rel_16		ATOMIC_SAN(clear_rel_16)
+#define	atomic_cmpset_16		ATOMIC_SAN(cmpset_16)
+#define	atomic_cmpset_acq_16		ATOMIC_SAN(cmpset_acq_16)
+#define	atomic_cmpset_rel_16		ATOMIC_SAN(cmpset_rel_16)
+#define	atomic_fcmpset_16		ATOMIC_SAN(fcmpset_16)
+#define	atomic_fcmpset_acq_16		ATOMIC_SAN(fcmpset_acq_16)
+#define	atomic_fcmpset_rel_16		ATOMIC_SAN(fcmpset_rel_16)
+#define	atomic_fetchadd_16		ATOMIC_SAN(fetchadd_16)
+#define	atomic_load_16			ATOMIC_SAN(load_16)
+#define	atomic_load_acq_16		ATOMIC_SAN(load_acq_16)
+#define	atomic_readandclear_16		ATOMIC_SAN(readandclear_16)
+#define	atomic_set_16			ATOMIC_SAN(set_16)
+#define	atomic_set_acq_16		ATOMIC_SAN(set_acq_16)
+#define	atomic_set_rel_16		ATOMIC_SAN(set_rel_16)
+#define	atomic_subtract_16		ATOMIC_SAN(subtract_16)
+#define	atomic_subtract_acq_16		ATOMIC_SAN(subtract_acq_16)
+#define	atomic_subtract_rel_16		ATOMIC_SAN(subtract_rel_16)
+#define	atomic_store_16			ATOMIC_SAN(store_16)
+#define	atomic_store_rel_16		ATOMIC_SAN(store_rel_16)
+#define	atomic_swap_16			ATOMIC_SAN(swap_16)
+#define	atomic_testandclear_16		ATOMIC_SAN(testandclear_16)
+#define	atomic_testandset_16		ATOMIC_SAN(testandset_16)
 
-#define	atomic_add_32			kcsan_atomic_add_32
-#define	atomic_add_acq_32		kcsan_atomic_add_acq_32
-#define	atomic_add_rel_32		kcsan_atomic_add_rel_32
-#define	atomic_clear_32			kcsan_atomic_clear_32
-#define	atomic_clear_acq_32		kcsan_atomic_clear_acq_32
-#define	atomic_clear_rel_32		kcsan_atomic_clear_rel_32
-#define	atomic_cmpset_32		kcsan_atomic_cmpset_32
-#define	atomic_cmpset_acq_32		kcsan_atomic_cmpset_acq_32
-#define	atomic_cmpset_rel_32		kcsan_atomic_cmpset_rel_32
-#define	atomic_fcmpset_32		kcsan_atomic_fcmpset_32
-#define	atomic_fcmpset_acq_32		kcsan_atomic_fcmpset_acq_32
-#define	atomic_fcmpset_rel_32		kcsan_atomic_fcmpset_rel_32
-#define	atomic_fetchadd_32		kcsan_atomic_fetchadd_32
-#define	atomic_load_32			kcsan_atomic_load_32
-#define	atomic_load_acq_32		kcsan_atomic_load_acq_32
-#define	atomic_readandclear_32		kcsan_atomic_readandclear_32
-#define	atomic_set_32			kcsan_atomic_set_32
-#define	atomic_set_acq_32		kcsan_atomic_set_acq_32
-#define	atomic_set_rel_32		kcsan_atomic_set_rel_32
-#define	atomic_subtract_32		kcsan_atomic_subtract_32
-#define	atomic_subtract_acq_32		kcsan_atomic_subtract_acq_32
-#define	atomic_subtract_rel_32		kcsan_atomic_subtract_rel_32
-#define	atomic_store_32			kcsan_atomic_store_32
-#define	atomic_store_rel_32		kcsan_atomic_store_rel_32
-#define	atomic_swap_32			kcsan_atomic_swap_32
-#define	atomic_testandclear_32		kcsan_atomic_testandclear_32
-#define	atomic_testandset_32		kcsan_atomic_testandset_32
+#define	atomic_add_32			ATOMIC_SAN(add_32)
+#define	atomic_add_acq_32		ATOMIC_SAN(add_acq_32)
+#define	atomic_add_rel_32		ATOMIC_SAN(add_rel_32)
+#define	atomic_clear_32			ATOMIC_SAN(clear_32)
+#define	atomic_clear_acq_32		ATOMIC_SAN(clear_acq_32)
+#define	atomic_clear_rel_32		ATOMIC_SAN(clear_rel_32)
+#define	atomic_cmpset_32		ATOMIC_SAN(cmpset_32)
+#define	atomic_cmpset_acq_32		ATOMIC_SAN(cmpset_acq_32)
+#define	atomic_cmpset_rel_32		ATOMIC_SAN(cmpset_rel_32)
+#define	atomic_fcmpset_32		ATOMIC_SAN(fcmpset_32)
+#define	atomic_fcmpset_acq_32		ATOMIC_SAN(fcmpset_acq_32)
+#define	atomic_fcmpset_rel_32		ATOMIC_SAN(fcmpset_rel_32)
+#define	atomic_fetchadd_32		ATOMIC_SAN(fetchadd_32)
+#define	atomic_load_32			ATOMIC_SAN(load_32)
+#define	atomic_load_acq_32		ATOMIC_SAN(load_acq_32)
+#define	atomic_readandclear_32		ATOMIC_SAN(readandclear_32)
+#define	atomic_set_32			ATOMIC_SAN(set_32)
+#define	atomic_set_acq_32		ATOMIC_SAN(set_acq_32)
+#define	atomic_set_rel_32		ATOMIC_SAN(set_rel_32)
+#define	atomic_subtract_32		ATOMIC_SAN(subtract_32)
+#define	atomic_subtract_acq_32		ATOMIC_SAN(subtract_acq_32)
+#define	atomic_subtract_rel_32		ATOMIC_SAN(subtract_rel_32)
+#define	atomic_store_32			ATOMIC_SAN(store_32)
+#define	atomic_store_rel_32		ATOMIC_SAN(store_rel_32)
+#define	atomic_swap_32			ATOMIC_SAN(swap_32)
+#define	atomic_testandclear_32		ATOMIC_SAN(testandclear_32)
+#define	atomic_testandset_32		ATOMIC_SAN(testandset_32)
 
-#define	atomic_add_64			kcsan_atomic_add_64
-#define	atomic_add_acq_64		kcsan_atomic_add_acq_64
-#define	atomic_add_rel_64		kcsan_atomic_add_rel_64
-#define	atomic_clear_64			kcsan_atomic_clear_64
-#define	atomic_clear_acq_64		kcsan_atomic_clear_acq_64
-#define	atomic_clear_rel_64		kcsan_atomic_clear_rel_64
-#define	atomic_cmpset_64		kcsan_atomic_cmpset_64
-#define	atomic_cmpset_acq_64		kcsan_atomic_cmpset_acq_64
-#define	atomic_cmpset_rel_64		kcsan_atomic_cmpset_rel_64
-#define	atomic_fcmpset_64		kcsan_atomic_fcmpset_64
-#define	atomic_fcmpset_acq_64		kcsan_atomic_fcmpset_acq_64
-#define	atomic_fcmpset_rel_64		kcsan_atomic_fcmpset_rel_64
-#define	atomic_fetchadd_64		kcsan_atomic_fetchadd_64
-#define	atomic_load_64			kcsan_atomic_load_64
-#define	atomic_load_acq_64		kcsan_atomic_load_acq_64
-#define	atomic_readandclear_64		kcsan_atomic_readandclear_64
-#define	atomic_set_64			kcsan_atomic_set_64
-#define	atomic_set_acq_64		kcsan_atomic_set_acq_64
-#define	atomic_set_rel_64		kcsan_atomic_set_rel_64
-#define	atomic_subtract_64		kcsan_atomic_subtract_64
-#define	atomic_subtract_acq_64		kcsan_atomic_subtract_acq_64
-#define	atomic_subtract_rel_64		kcsan_atomic_subtract_rel_64
-#define	atomic_store_64			kcsan_atomic_store_64
-#define	atomic_store_rel_64		kcsan_atomic_store_rel_64
-#define	atomic_swap_64			kcsan_atomic_swap_64
-#define	atomic_testandclear_64		kcsan_atomic_testandclear_64
-#define	atomic_testandset_64		kcsan_atomic_testandset_64
+#define	atomic_add_64			ATOMIC_SAN(add_64)
+#define	atomic_add_acq_64		ATOMIC_SAN(add_acq_64)
+#define	atomic_add_rel_64		ATOMIC_SAN(add_rel_64)
+#define	atomic_clear_64			ATOMIC_SAN(clear_64)
+#define	atomic_clear_acq_64		ATOMIC_SAN(clear_acq_64)
+#define	atomic_clear_rel_64		ATOMIC_SAN(clear_rel_64)
+#define	atomic_cmpset_64		ATOMIC_SAN(cmpset_64)
+#define	atomic_cmpset_acq_64		ATOMIC_SAN(cmpset_acq_64)
+#define	atomic_cmpset_rel_64		ATOMIC_SAN(cmpset_rel_64)
+#define	atomic_fcmpset_64		ATOMIC_SAN(fcmpset_64)
+#define	atomic_fcmpset_acq_64		ATOMIC_SAN(fcmpset_acq_64)
+#define	atomic_fcmpset_rel_64		ATOMIC_SAN(fcmpset_rel_64)
+#define	atomic_fetchadd_64		ATOMIC_SAN(fetchadd_64)
+#define	atomic_load_64			ATOMIC_SAN(load_64)
+#define	atomic_load_acq_64		ATOMIC_SAN(load_acq_64)
+#define	atomic_readandclear_64		ATOMIC_SAN(readandclear_64)
+#define	atomic_set_64			ATOMIC_SAN(set_64)
+#define	atomic_set_acq_64		ATOMIC_SAN(set_acq_64)
+#define	atomic_set_rel_64		ATOMIC_SAN(set_rel_64)
+#define	atomic_subtract_64		ATOMIC_SAN(subtract_64)
+#define	atomic_subtract_acq_64		ATOMIC_SAN(subtract_acq_64)
+#define	atomic_subtract_rel_64		ATOMIC_SAN(subtract_rel_64)
+#define	atomic_store_64			ATOMIC_SAN(store_64)
+#define	atomic_store_rel_64		ATOMIC_SAN(store_rel_64)
+#define	atomic_swap_64			ATOMIC_SAN(swap_64)
+#define	atomic_testandclear_64		ATOMIC_SAN(testandclear_64)
+#define	atomic_testandset_64		ATOMIC_SAN(testandset_64)
 
-#define	atomic_thread_fence_acq		kcsan_atomic_thread_fence_acq
-#define	atomic_thread_fence_acq_rel	kcsan_atomic_thread_fence_acq_rel
-#define	atomic_thread_fence_rel		kcsan_atomic_thread_fence_rel
-#define	atomic_thread_fence_seq_cst	kcsan_atomic_thread_fence_seq_cst
-#define	atomic_interrupt_fence		__compiler_membar
+#define	atomic_thread_fence_acq		ATOMIC_SAN(thread_fence_acq)
+#define	atomic_thread_fence_acq_rel	ATOMIC_SAN(thread_fence_acq_rel)
+#define	atomic_thread_fence_rel		ATOMIC_SAN(thread_fence_rel)
+#define	atomic_thread_fence_seq_cst	ATOMIC_SAN(thread_fence_seq_cst)
+#define	atomic_interrupt_fence		ATOMIC_SAN(interrupt_fence)
 
-#endif /* !KCSAN_RUNTIME */
+#endif /* !SAN_RUNTIME */
 
 #endif /* !_SYS_ATOMIC_SAN_H_ */
diff --git a/sys/sys/bus_san.h b/sys/sys/bus_san.h
index 96b2441fbdbe..05d5ecd4b844 100644
--- a/sys/sys/bus_san.h
+++ b/sys/sys/bus_san.h
@@ -2,12 +2,16 @@
  * SPDX-License-Identifier: BSD-2-Clause
  *
  * Copyright (c) 2019 Andrew Turner
+ * Copyright (c) 2021 The FreeBSD Foundation
  *
  * This software was developed by SRI International and the University of
  * Cambridge Computer Laboratory (Department of Computer Science and
  * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
  * DARPA SSITH research programme.
  *
+ * Portions of this software were written by Mark Johnston under sponsorship by
+ * the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -35,174 +39,190 @@
 #ifndef _SYS_BUS_SAN_H_
 #define	_SYS_BUS_SAN_H_
 
-#define	KCSAN_BS_MULTI(rw, width, type)					\
-	void kcsan_bus_space_##rw##_multi_##width(bus_space_tag_t, 	\
+#ifndef _MACHINE_BUS_H_
+#error do not include this header, use machine/bus.h
+#endif
+
+#ifndef BUS_SAN_PREFIX
+#error No sanitizer prefix defined
+#endif
+
+#define	BUS_SAN_MULTI(sp, rw, width, type)				\
+	void sp##_bus_space_##rw##_multi_##width(bus_space_tag_t, 	\
 	    bus_space_handle_t, bus_size_t, type *, bus_size_t);	\
-	void kcsan_bus_space_##rw##_multi_stream_##width(bus_space_tag_t, \
+	void sp##_bus_space_##rw##_multi_stream_##width(bus_space_tag_t, \
 	    bus_space_handle_t, bus_size_t, type *, bus_size_t);	\
-	void kcsan_bus_space_##rw##_region_##width(bus_space_tag_t,	\
+	void sp##_bus_space_##rw##_region_##width(bus_space_tag_t,	\
 	    bus_space_handle_t, bus_size_t, type *, bus_size_t);	\
-	void kcsan_bus_space_##rw##_region_stream_##width(bus_space_tag_t, \
+	void sp##_bus_space_##rw##_region_stream_##width(bus_space_tag_t, \
 	    bus_space_handle_t, bus_size_t, type *, bus_size_t)
 
-#define	KCSAN_BS_READ(width, type)					\
-	type kcsan_bus_space_read_##width(bus_space_tag_t, 		\
+#define	BUS_SAN_READ(sp, width, type)					\
+	type sp##_bus_space_read_##width(bus_space_tag_t, 		\
 	    bus_space_handle_t, bus_size_t);				\
-	type kcsan_bus_space_read_stream_##width(bus_space_tag_t,	\
+	type sp##_bus_space_read_stream_##width(bus_space_tag_t,	\
 	    bus_space_handle_t, bus_size_t);				\
-	KCSAN_BS_MULTI(read, width, type)
+	BUS_SAN_MULTI(sp, read, width, type)
 
-#define	KCSAN_BS_WRITE(width, type)					\
-	void kcsan_bus_space_write_##width(bus_space_tag_t, 		\
+#define	BUS_SAN_WRITE(sp, width, type)					\
+	void sp##_bus_space_write_##width(bus_space_tag_t, 		\
 	    bus_space_handle_t, bus_size_t, type);			\
-	void kcsan_bus_space_write_stream_##width(bus_space_tag_t,	\
+	void sp##_bus_space_write_stream_##width(bus_space_tag_t,	\
 	    bus_space_handle_t, bus_size_t, type);			\
-	KCSAN_BS_MULTI(write, width, const type)
+	BUS_SAN_MULTI(sp, write, width, const type)
 
-#define	KCSAN_BS_SET(width, type)					\
-	void kcsan_bus_space_set_multi_##width(bus_space_tag_t, 	\
+#define	BUS_SAN_SET(sp, width, type)					\
+	void sp##_bus_space_set_multi_##width(bus_space_tag_t,		\
 	    bus_space_handle_t, bus_size_t, type, bus_size_t);		\
-	void kcsan_bus_space_set_multi_stream_##width(bus_space_tag_t, \
+	void sp##_bus_space_set_multi_stream_##width(bus_space_tag_t,	\
 	    bus_space_handle_t, bus_size_t, type, bus_size_t);		\
-	void kcsan_bus_space_set_region_##width(bus_space_tag_t,	\
+	void sp##_bus_space_set_region_##width(bus_space_tag_t,		\
 	    bus_space_handle_t, bus_size_t, type, bus_size_t);		\
-	void kcsan_bus_space_set_region_stream_##width(bus_space_tag_t, \
+	void sp##_bus_space_set_region_stream_##width(bus_space_tag_t,	\
 	    bus_space_handle_t, bus_size_t, type, bus_size_t)
 
-#define	KCSAN_BS_COPY(width, type)					\
-	void kcsan_bus_space_copy_region_##width(bus_space_tag_t,	\
+#define	BUS_SAN_COPY(sp, width, type)					\
+	void sp##_bus_space_copy_region_##width(bus_space_tag_t,	\
 	    bus_space_handle_t,	bus_size_t, bus_space_handle_t,		\
 	    bus_size_t, bus_size_t);					\
-	void kcsan_bus_space_copy_region_stream_##width(bus_space_tag_t, \
+	void sp##_bus_space_copy_region_stream_##width(bus_space_tag_t, \
 	    bus_space_handle_t,	bus_size_t, bus_space_handle_t,		\
 	    bus_size_t, bus_size_t);
 
-#define	KCSAN_BS_PEEK(width, type)					\
-	int kcsan_bus_space_peek_##width(bus_space_tag_t, 		\
+#define	BUS_SAN_PEEK(sp, width, type)					\
+	int sp##_bus_space_peek_##width(bus_space_tag_t, 		\
 	    bus_space_handle_t, bus_size_t, type *);
 
-#define	KCSAN_BS_POKE(width, type)					\
-	int kcsan_bus_space_poke_##width(bus_space_tag_t, 		\
+#define	BUS_SAN_POKE(sp, width, type)					\
+	int sp##_bus_space_poke_##width(bus_space_tag_t, 		\
 	    bus_space_handle_t, bus_size_t, type);
 
-#define	KCSAN_BS(width, type)						\
-	KCSAN_BS_READ(width, type);					\
-	KCSAN_BS_WRITE(width, type);					\
-	KCSAN_BS_SET(width, type);					\
-	KCSAN_BS_COPY(width, type)					\
-	KCSAN_BS_PEEK(width, type);					\
-	KCSAN_BS_POKE(width, type);
-
-KCSAN_BS(1, uint8_t);
-KCSAN_BS(2, uint16_t);
-KCSAN_BS(4, uint32_t);
-KCSAN_BS(8, uint64_t);
-
-int kcsan_bus_space_map(bus_space_tag_t, bus_addr_t, bus_size_t, int,
-    bus_space_handle_t *);
-void kcsan_bus_space_unmap(bus_space_tag_t, bus_space_handle_t, bus_size_t);
-int kcsan_bus_space_subregion(bus_space_tag_t, bus_space_handle_t, bus_size_t,
-    bus_size_t, bus_space_handle_t *);
-int kcsan_bus_space_alloc(bus_space_tag_t, bus_addr_t, bus_addr_t,
*** 284 LINES SKIPPED ***



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202111011433.1A1EXICa021796>