Skip site navigation (1)Skip section navigation (2)
Date:      Sat, 16 Jun 2018 08:26:24 +0000 (UTC)
From:      "Andrey V. Elsukov" <ae@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r335250 - in head/sys: kern net netinet netinet6 netpfil/ipfw nfs
Message-ID:  <201806160826.w5G8QOLU008115@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: ae
Date: Sat Jun 16 08:26:23 2018
New Revision: 335250
URL: https://svnweb.freebsd.org/changeset/base/335250

Log:
  Switch RIB and RADIX_NODE_HEAD lock from rwlock(9) to rmlock(9).
  
  Using of rwlock with multiqueue NICs for IP forwarding on high pps
  produces high lock contention and inefficient. Rmlock fits better for
  such workloads.
  
  Reviewed by:	melifaro, olivier
  Obtained from:	Yandex LLC
  Sponsored by:	Yandex LLC
  Differential Revision:	https://reviews.freebsd.org/D15789

Modified:
  head/sys/kern/subr_witness.c
  head/sys/kern/vfs_export.c
  head/sys/net/radix.c
  head/sys/net/radix.h
  head/sys/net/radix_mpath.c
  head/sys/net/route.c
  head/sys/net/route_var.h
  head/sys/net/rtsock.c
  head/sys/netinet/in_fib.c
  head/sys/netinet6/in6_fib.c
  head/sys/netpfil/ipfw/ip_fw_table_algo.c
  head/sys/nfs/bootp_subr.c

Modified: head/sys/kern/subr_witness.c
==============================================================================
--- head/sys/kern/subr_witness.c	Sat Jun 16 08:25:38 2018	(r335249)
+++ head/sys/kern/subr_witness.c	Sat Jun 16 08:26:23 2018	(r335250)
@@ -524,7 +524,7 @@ static struct witness_order_list_entry order_lists[] =
 	 * Routing
 	 */
 	{ "so_rcv", &lock_class_mtx_sleep },
-	{ "radix node head", &lock_class_rw },
+	{ "radix node head", &lock_class_rm },
 	{ "rtentry", &lock_class_mtx_sleep },
 	{ "ifaddr", &lock_class_mtx_sleep },
 	{ NULL, NULL },

Modified: head/sys/kern/vfs_export.c
==============================================================================
--- head/sys/kern/vfs_export.c	Sat Jun 16 08:25:38 2018	(r335249)
+++ head/sys/kern/vfs_export.c	Sat Jun 16 08:26:23 2018	(r335250)
@@ -51,7 +51,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/mbuf.h>
 #include <sys/mount.h>
 #include <sys/mutex.h>
-#include <sys/rwlock.h>
+#include <sys/rmlock.h>
 #include <sys/refcount.h>
 #include <sys/signalvar.h>
 #include <sys/socket.h>
@@ -449,6 +449,7 @@ vfs_setpublicfs(struct mount *mp, struct netexport *ne
 static struct netcred *
 vfs_export_lookup(struct mount *mp, struct sockaddr *nam)
 {
+	RADIX_NODE_HEAD_RLOCK_TRACKER;
 	struct netexport *nep;
 	struct netcred *np = NULL;
 	struct radix_node_head *rnh;

Modified: head/sys/net/radix.c
==============================================================================
--- head/sys/net/radix.c	Sat Jun 16 08:25:38 2018	(r335249)
+++ head/sys/net/radix.c	Sat Jun 16 08:26:23 2018	(r335250)
@@ -39,7 +39,7 @@
 #ifdef	_KERNEL
 #include <sys/lock.h>
 #include <sys/mutex.h>
-#include <sys/rwlock.h>
+#include <sys/rmlock.h>
 #include <sys/systm.h>
 #include <sys/malloc.h>
 #include <sys/syslog.h>

Modified: head/sys/net/radix.h
==============================================================================
--- head/sys/net/radix.h	Sat Jun 16 08:25:38 2018	(r335249)
+++ head/sys/net/radix.h	Sat Jun 16 08:26:23 2018	(r335250)
@@ -38,7 +38,7 @@
 #ifdef _KERNEL
 #include <sys/_lock.h>
 #include <sys/_mutex.h>
-#include <sys/_rwlock.h>
+#include <sys/_rmlock.h>
 #endif
 
 #ifdef MALLOC_DECLARE
@@ -138,7 +138,7 @@ struct radix_node_head {
 	rn_close_t	*rnh_close;	/*do something when the last ref drops*/
 	struct	radix_node rnh_nodes[3];	/* empty tree for common case */
 #ifdef _KERNEL
-	struct	rwlock rnh_lock;		/* locks entire radix tree */
+	struct	rmlock rnh_lock;		/* locks entire radix tree */
 #endif
 };
 
@@ -159,18 +159,18 @@ void rn_inithead_internal(struct radix_head *rh, struc
 #define R_Zalloc(p, t, n) (p = (t) malloc((unsigned long)(n), M_RTABLE, M_NOWAIT | M_ZERO))
 #define R_Free(p) free((caddr_t)p, M_RTABLE);
 
+#define	RADIX_NODE_HEAD_RLOCK_TRACKER	struct rm_priotracker _rhn_tracker
 #define	RADIX_NODE_HEAD_LOCK_INIT(rnh)	\
-    rw_init_flags(&(rnh)->rnh_lock, "radix node head", 0)
-#define	RADIX_NODE_HEAD_LOCK(rnh)	rw_wlock(&(rnh)->rnh_lock)
-#define	RADIX_NODE_HEAD_UNLOCK(rnh)	rw_wunlock(&(rnh)->rnh_lock)
-#define	RADIX_NODE_HEAD_RLOCK(rnh)	rw_rlock(&(rnh)->rnh_lock)
-#define	RADIX_NODE_HEAD_RUNLOCK(rnh)	rw_runlock(&(rnh)->rnh_lock)
-#define	RADIX_NODE_HEAD_LOCK_TRY_UPGRADE(rnh)	rw_try_upgrade(&(rnh)->rnh_lock)
-
-
-#define	RADIX_NODE_HEAD_DESTROY(rnh)	rw_destroy(&(rnh)->rnh_lock)
-#define	RADIX_NODE_HEAD_LOCK_ASSERT(rnh) rw_assert(&(rnh)->rnh_lock, RA_LOCKED)
-#define	RADIX_NODE_HEAD_WLOCK_ASSERT(rnh) rw_assert(&(rnh)->rnh_lock, RA_WLOCKED)
+    rm_init(&(rnh)->rnh_lock, "radix node head")
+#define	RADIX_NODE_HEAD_LOCK(rnh)	rm_wlock(&(rnh)->rnh_lock)
+#define	RADIX_NODE_HEAD_UNLOCK(rnh)	rm_wunlock(&(rnh)->rnh_lock)
+#define	RADIX_NODE_HEAD_RLOCK(rnh)	rm_rlock(&(rnh)->rnh_lock,\
+    &_rhn_tracker)
+#define	RADIX_NODE_HEAD_RUNLOCK(rnh)	rm_runlock(&(rnh)->rnh_lock,\
+    &_rhn_tracker)
+#define	RADIX_NODE_HEAD_DESTROY(rnh)	rm_destroy(&(rnh)->rnh_lock)
+#define	RADIX_NODE_HEAD_LOCK_ASSERT(rnh) rm_assert(&(rnh)->rnh_lock, RA_LOCKED)
+#define	RADIX_NODE_HEAD_WLOCK_ASSERT(rnh) rm_assert(&(rnh)->rnh_lock, RA_WLOCKED)
 #endif /* _KERNEL */
 
 int	 rn_inithead(void **, int);

Modified: head/sys/net/radix_mpath.c
==============================================================================
--- head/sys/net/radix_mpath.c	Sat Jun 16 08:25:38 2018	(r335249)
+++ head/sys/net/radix_mpath.c	Sat Jun 16 08:26:23 2018	(r335250)
@@ -43,12 +43,15 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/lock.h>
 #include <sys/malloc.h>
+#include <sys/mutex.h>
 #include <sys/socket.h>
 #include <sys/domain.h>
 #include <sys/syslog.h>
 #include <net/radix.h>
 #include <net/radix_mpath.h>
+#include <sys/rmlock.h>
 #include <net/route.h>
 #include <net/route_var.h>
 #include <net/if.h>

Modified: head/sys/net/route.c
==============================================================================
--- head/sys/net/route.c	Sat Jun 16 08:25:38 2018	(r335249)
+++ head/sys/net/route.c	Sat Jun 16 08:26:23 2018	(r335250)
@@ -54,6 +54,8 @@
 #include <sys/proc.h>
 #include <sys/domain.h>
 #include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/rmlock.h>
 
 #include <net/if.h>
 #include <net/if_var.h>
@@ -440,6 +442,7 @@ struct rtentry *
 rtalloc1_fib(struct sockaddr *dst, int report, u_long ignflags,
 		    u_int fibnum)
 {
+	RIB_RLOCK_TRACKER;
 	struct rib_head *rh;
 	struct radix_node *rn;
 	struct rtentry *newrt;
@@ -923,6 +926,7 @@ int
 rib_lookup_info(uint32_t fibnum, const struct sockaddr *dst, uint32_t flags,
     uint32_t flowid, struct rt_addrinfo *info)
 {
+	RIB_RLOCK_TRACKER;
 	struct rib_head *rh;
 	struct radix_node *rn;
 	struct rtentry *rt;
@@ -1944,6 +1948,7 @@ rt_maskedcopy(struct sockaddr *src, struct sockaddr *d
 static inline  int
 rtinit1(struct ifaddr *ifa, int cmd, int flags, int fibnum)
 {
+	RIB_RLOCK_TRACKER;
 	struct sockaddr *dst;
 	struct sockaddr *netmask;
 	struct rtentry *rt = NULL;

Modified: head/sys/net/route_var.h
==============================================================================
--- head/sys/net/route_var.h	Sat Jun 16 08:25:38 2018	(r335249)
+++ head/sys/net/route_var.h	Sat Jun 16 08:26:23 2018	(r335250)
@@ -44,18 +44,19 @@ struct rib_head {
 	rt_gen_t		rnh_gen;	/* generation counter */
 	int			rnh_multipath;	/* multipath capable ? */
 	struct radix_node	rnh_nodes[3];	/* empty tree for common case */
-	struct rwlock		rib_lock;	/* config/data path lock */
+	struct rmlock		rib_lock;	/* config/data path lock */
 	struct radix_mask_head	rmhead;		/* masks radix head */
 };
 
-#define	RIB_LOCK_INIT(rh)	rw_init(&(rh)->rib_lock, "rib head lock")
-#define	RIB_LOCK_DESTROY(rh)	rw_destroy(&(rh)->rib_lock)
-#define	RIB_RLOCK(rh)		rw_rlock(&(rh)->rib_lock)
-#define	RIB_RUNLOCK(rh)		rw_runlock(&(rh)->rib_lock)
-#define	RIB_WLOCK(rh)		rw_wlock(&(rh)->rib_lock)
-#define	RIB_WUNLOCK(rh)		rw_wunlock(&(rh)->rib_lock)
-#define	RIB_LOCK_ASSERT(rh)	rw_assert(&(rh)->rib_lock, RA_LOCKED)
-#define	RIB_WLOCK_ASSERT(rh)	rw_assert(&(rh)->rib_lock, RA_WLOCKED)
+#define	RIB_RLOCK_TRACKER	struct rm_priotracker _rib_tracker
+#define	RIB_LOCK_INIT(rh)	rm_init(&(rh)->rib_lock, "rib head lock")
+#define	RIB_LOCK_DESTROY(rh)	rm_destroy(&(rh)->rib_lock)
+#define	RIB_RLOCK(rh)		rm_rlock(&(rh)->rib_lock, &_rib_tracker)
+#define	RIB_RUNLOCK(rh)		rm_runlock(&(rh)->rib_lock, &_rib_tracker)
+#define	RIB_WLOCK(rh)		rm_wlock(&(rh)->rib_lock)
+#define	RIB_WUNLOCK(rh)		rm_wunlock(&(rh)->rib_lock)
+#define	RIB_LOCK_ASSERT(rh)	rm_assert(&(rh)->rib_lock, RA_LOCKED)
+#define	RIB_WLOCK_ASSERT(rh)	rm_assert(&(rh)->rib_lock, RA_WLOCKED)
 
 struct rib_head *rt_tables_get_rnh(int fib, int family);
 

Modified: head/sys/net/rtsock.c
==============================================================================
--- head/sys/net/rtsock.c	Sat Jun 16 08:25:38 2018	(r335249)
+++ head/sys/net/rtsock.c	Sat Jun 16 08:26:23 2018	(r335250)
@@ -45,6 +45,7 @@
 #include <sys/priv.h>
 #include <sys/proc.h>
 #include <sys/protosw.h>
+#include <sys/rmlock.h>
 #include <sys/rwlock.h>
 #include <sys/signalvar.h>
 #include <sys/socket.h>
@@ -542,6 +543,7 @@ rtm_get_jailed(struct rt_addrinfo *info, struct ifnet 
 static int
 route_output(struct mbuf *m, struct socket *so, ...)
 {
+	RIB_RLOCK_TRACKER;
 	struct rt_msghdr *rtm = NULL;
 	struct rtentry *rt = NULL;
 	struct rib_head *rnh;
@@ -1850,6 +1852,7 @@ sysctl_ifmalist(int af, struct walkarg *w)
 static int
 sysctl_rtsock(SYSCTL_HANDLER_ARGS)
 {
+	RIB_RLOCK_TRACKER;
 	int	*name = (int *)arg1;
 	u_int	namelen = arg2;
 	struct rib_head *rnh = NULL; /* silence compiler. */

Modified: head/sys/netinet/in_fib.c
==============================================================================
--- head/sys/netinet/in_fib.c	Sat Jun 16 08:25:38 2018	(r335249)
+++ head/sys/netinet/in_fib.c	Sat Jun 16 08:26:23 2018	(r335250)
@@ -37,7 +37,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/lock.h>
-#include <sys/rwlock.h>
+#include <sys/rmlock.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/socket.h>
@@ -134,6 +134,7 @@ int
 fib4_lookup_nh_basic(uint32_t fibnum, struct in_addr dst, uint32_t flags,
     uint32_t flowid, struct nhop4_basic *pnh4)
 {
+	RIB_RLOCK_TRACKER;
 	struct rib_head *rh;
 	struct radix_node *rn;
 	struct sockaddr_in sin;
@@ -182,6 +183,7 @@ int
 fib4_lookup_nh_ext(uint32_t fibnum, struct in_addr dst, uint32_t flags,
     uint32_t flowid, struct nhop4_extended *pnh4)
 {
+	RIB_RLOCK_TRACKER;
 	struct rib_head *rh;
 	struct radix_node *rn;
 	struct sockaddr_in sin;

Modified: head/sys/netinet6/in6_fib.c
==============================================================================
--- head/sys/netinet6/in6_fib.c	Sat Jun 16 08:25:38 2018	(r335249)
+++ head/sys/netinet6/in6_fib.c	Sat Jun 16 08:26:23 2018	(r335250)
@@ -38,7 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/lock.h>
-#include <sys/rwlock.h>
+#include <sys/rmlock.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/socket.h>
@@ -171,6 +171,7 @@ int
 fib6_lookup_nh_basic(uint32_t fibnum, const struct in6_addr *dst, uint32_t scopeid,
     uint32_t flags, uint32_t flowid, struct nhop6_basic *pnh6)
 {
+	RIB_RLOCK_TRACKER;
 	struct rib_head *rh;
 	struct radix_node *rn;
 	struct sockaddr_in6 sin6;
@@ -220,6 +221,7 @@ int
 fib6_lookup_nh_ext(uint32_t fibnum, const struct in6_addr *dst,uint32_t scopeid,
     uint32_t flags, uint32_t flowid, struct nhop6_extended *pnh6)
 {
+	RIB_RLOCK_TRACKER;
 	struct rib_head *rh;
 	struct radix_node *rn;
 	struct sockaddr_in6 sin6;

Modified: head/sys/netpfil/ipfw/ip_fw_table_algo.c
==============================================================================
--- head/sys/netpfil/ipfw/ip_fw_table_algo.c	Sat Jun 16 08:25:38 2018	(r335249)
+++ head/sys/netpfil/ipfw/ip_fw_table_algo.c	Sat Jun 16 08:26:23 2018	(r335250)
@@ -4047,6 +4047,7 @@ static void
 ta_foreach_kfib(void *ta_state, struct table_info *ti, ta_foreach_f *f,
     void *arg)
 {
+	RIB_RLOCK_TRACKER;
 	struct rib_head *rh;
 	int error;
 

Modified: head/sys/nfs/bootp_subr.c
==============================================================================
--- head/sys/nfs/bootp_subr.c	Sat Jun 16 08:25:38 2018	(r335249)
+++ head/sys/nfs/bootp_subr.c	Sat Jun 16 08:26:23 2018	(r335250)
@@ -376,6 +376,7 @@ bootpboot_p_tree(struct radix_node *rn)
 void
 bootpboot_p_rtlist(void)
 {
+	RIB_RLOCK_TRACKER;
 	struct rib_head *rnh;
 
 	printf("Routing table:\n");



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201806160826.w5G8QOLU008115>