Skip site navigation (1)Skip section navigation (2)
Date:      Sun, 26 Apr 2020 16:22:35 +0000 (UTC)
From:      Kristof Provost <kp@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r360345 - head/sys/net
Message-ID:  <202004261622.03QGMZWt001811@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: kp
Date: Sun Apr 26 16:22:35 2020
New Revision: 360345
URL: https://svnweb.freebsd.org/changeset/base/360345

Log:
  bridge: epoch-ification
  
  Run the bridge datapath under epoch, rather than under the
  BRIDGE_LOCK().
  
  We still take the BRIDGE_LOCK() whenever we insert or delete items in
  the relevant lists, but we use epoch callbacks to free items so that
  it's safe to iterate the lists without the BRIDGE_LOCK.
  
  Tests on mercat5/6 shows this increases bridge throughput significantly,
  from 3.7Mpps to 18.6Mpps.
  
  Reviewed by:	emaste, philip, melifaro
  MFC after:	2 months
  Sponsored by:	The FreeBSD Foundation
  Differential Revision:	https://reviews.freebsd.org/D24250

Modified:
  head/sys/net/if_bridge.c

Modified: head/sys/net/if_bridge.c
==============================================================================
--- head/sys/net/if_bridge.c	Sun Apr 26 16:16:39 2020	(r360344)
+++ head/sys/net/if_bridge.c	Sun Apr 26 16:22:35 2020	(r360345)
@@ -189,41 +189,14 @@ extern void	nd6_setmtu(struct ifnet *);
  */
 #define BRIDGE_LOCK_INIT(_sc)		do {			\
 	mtx_init(&(_sc)->sc_mtx, "if_bridge", NULL, MTX_DEF);	\
-	cv_init(&(_sc)->sc_cv, "if_bridge_cv");			\
 } while (0)
 #define BRIDGE_LOCK_DESTROY(_sc)	do {	\
 	mtx_destroy(&(_sc)->sc_mtx);		\
-	cv_destroy(&(_sc)->sc_cv);		\
 } while (0)
 #define BRIDGE_LOCK(_sc)		mtx_lock(&(_sc)->sc_mtx)
 #define BRIDGE_UNLOCK(_sc)		mtx_unlock(&(_sc)->sc_mtx)
 #define BRIDGE_LOCK_ASSERT(_sc)		mtx_assert(&(_sc)->sc_mtx, MA_OWNED)
 #define BRIDGE_UNLOCK_ASSERT(_sc)	mtx_assert(&(_sc)->sc_mtx, MA_NOTOWNED)
-#define	BRIDGE_LOCK2REF(_sc, _err)	do {	\
-	mtx_assert(&(_sc)->sc_mtx, MA_OWNED);	\
-	if ((_sc)->sc_iflist_xcnt > 0)		\
-		(_err) = EBUSY;			\
-	else					\
-		(_sc)->sc_iflist_ref++;		\
-	mtx_unlock(&(_sc)->sc_mtx);		\
-} while (0)
-#define	BRIDGE_UNREF(_sc)		do {				\
-	mtx_lock(&(_sc)->sc_mtx);					\
-	(_sc)->sc_iflist_ref--;						\
-	if (((_sc)->sc_iflist_xcnt > 0) && ((_sc)->sc_iflist_ref == 0))	\
-		cv_broadcast(&(_sc)->sc_cv);				\
-	mtx_unlock(&(_sc)->sc_mtx);					\
-} while (0)
-#define	BRIDGE_XLOCK(_sc)		do {		\
-	mtx_assert(&(_sc)->sc_mtx, MA_OWNED);		\
-	(_sc)->sc_iflist_xcnt++;			\
-	while ((_sc)->sc_iflist_ref > 0)		\
-		cv_wait(&(_sc)->sc_cv, &(_sc)->sc_mtx);	\
-} while (0)
-#define	BRIDGE_XDROP(_sc)		do {	\
-	mtx_assert(&(_sc)->sc_mtx, MA_OWNED);	\
-	(_sc)->sc_iflist_xcnt--;		\
-} while (0)
 
 /*
  * Bridge interface list entry.
@@ -237,6 +210,7 @@ struct bridge_iflist {
 	uint32_t		bif_addrmax;	/* max # of addresses */
 	uint32_t		bif_addrcnt;	/* cur. # of addresses */
 	uint32_t		bif_addrexceeded;/* # of address violations */
+	struct epoch_context	bif_epoch_ctx;
 };
 
 /*
@@ -250,6 +224,8 @@ struct bridge_rtnode {
 	uint8_t			brt_flags;	/* address flags */
 	uint8_t			brt_addr[ETHER_ADDR_LEN];
 	uint16_t		brt_vlan;	/* vlan id */
+	struct	vnet		*brt_vnet;
+	struct	epoch_context	brt_epoch_ctx;
 };
 #define	brt_ifp			brt_dst->bif_ifp
 
@@ -260,13 +236,10 @@ struct bridge_softc {
 	struct ifnet		*sc_ifp;	/* make this an interface */
 	LIST_ENTRY(bridge_softc) sc_list;
 	struct mtx		sc_mtx;
-	struct cv		sc_cv;
 	uint32_t		sc_brtmax;	/* max # of addresses */
 	uint32_t		sc_brtcnt;	/* cur. # of addresses */
 	uint32_t		sc_brttimeout;	/* rt timeout in seconds */
 	struct callout		sc_brcallout;	/* bridge callout */
-	uint32_t		sc_iflist_ref;	/* refcount for sc_iflist */
-	uint32_t		sc_iflist_xcnt;	/* refcount for sc_iflist */
 	CK_LIST_HEAD(, bridge_iflist) sc_iflist;	/* member interface list */
 	CK_LIST_HEAD(, bridge_rtnode) *sc_rthash;	/* our forwarding table */
 	CK_LIST_HEAD(, bridge_rtnode) sc_rtlist;	/* list version of above */
@@ -276,6 +249,7 @@ struct bridge_softc {
 	uint32_t		sc_brtexceeded;	/* # of cache drops */
 	struct ifnet		*sc_ifaddr;	/* member mac copied from */
 	struct ether_addr	sc_defaddr;	/* Default MAC address */
+	struct epoch_context	sc_epoch_ctx;
 };
 
 VNET_DEFINE_STATIC(struct mtx, bridge_list_mtx);
@@ -596,6 +570,10 @@ vnet_bridge_uninit(const void *unused __unused)
 	if_clone_detach(V_bridge_cloner);
 	V_bridge_cloner = NULL;
 	BRIDGE_LIST_LOCK_DESTROY();
+
+	/* Callbacks may use the UMA zone. */
+	epoch_drain_callbacks(net_epoch_preempt);
+
 	uma_zdestroy(V_bridge_rtnode_zone);
 }
 VNET_SYSUNINIT(vnet_bridge_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY,
@@ -722,6 +700,17 @@ bridge_clone_create(struct if_clone *ifc, int unit, ca
 	return (0);
 }
 
+static void
+bridge_clone_destroy_cb(struct epoch_context *ctx)
+{
+	struct bridge_softc *sc;
+
+	sc = __containerof(ctx, struct bridge_softc, sc_epoch_ctx);
+
+	BRIDGE_LOCK_DESTROY(sc);
+	free(sc, M_DEVBUF);
+}
+
 /*
  * bridge_clone_destroy:
  *
@@ -732,7 +721,9 @@ bridge_clone_destroy(struct ifnet *ifp)
 {
 	struct bridge_softc *sc = ifp->if_softc;
 	struct bridge_iflist *bif;
+	struct epoch_tracker et;
 
+	NET_EPOCH_ENTER(et);
 	BRIDGE_LOCK(sc);
 
 	bridge_stop(ifp, 1);
@@ -757,11 +748,12 @@ bridge_clone_destroy(struct ifnet *ifp)
 	BRIDGE_LIST_UNLOCK();
 
 	bstp_detach(&sc->sc_stp);
+	NET_EPOCH_EXIT(et);
+
 	ether_ifdetach(ifp);
 	if_free(ifp);
 
-	BRIDGE_LOCK_DESTROY(sc);
-	free(sc, M_DEVBUF);
+	NET_EPOCH_CALL(bridge_clone_destroy_cb, &sc->sc_epoch_ctx);
 }
 
 /*
@@ -787,7 +779,10 @@ bridge_ioctl(struct ifnet *ifp, u_long cmd, caddr_t da
 	struct ifdrv *ifd = (struct ifdrv *) data;
 	const struct bridge_control *bc;
 	int error = 0, oldmtu;
+	struct epoch_tracker et;
 
+	NET_EPOCH_ENTER(et);
+
 	switch (cmd) {
 
 	case SIOCADDMULTI:
@@ -908,6 +903,8 @@ bridge_ioctl(struct ifnet *ifp, u_long cmd, caddr_t da
 		break;
 	}
 
+	NET_EPOCH_EXIT(et);
+
 	return (error);
 }
 
@@ -922,6 +919,8 @@ bridge_mutecaps(struct bridge_softc *sc)
 	struct bridge_iflist *bif;
 	int enabled, mask;
 
+	BRIDGE_LOCK_ASSERT(sc);
+
 	/* Initial bitmask of capabilities to test */
 	mask = BRIDGE_IFCAPS_MASK;
 
@@ -930,7 +929,6 @@ bridge_mutecaps(struct bridge_softc *sc)
 		mask &= bif->bif_savedcaps;
 	}
 
-	BRIDGE_XLOCK(sc);
 	CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) {
 		enabled = bif->bif_ifp->if_capenable;
 		enabled &= ~BRIDGE_IFCAPS_STRIP;
@@ -941,8 +939,6 @@ bridge_mutecaps(struct bridge_softc *sc)
 		bridge_set_ifcap(sc, bif, enabled);
 		BRIDGE_LOCK(sc);
 	}
-	BRIDGE_XDROP(sc);
-
 }
 
 static void
@@ -983,7 +979,7 @@ bridge_lookup_member(struct bridge_softc *sc, const ch
 	struct bridge_iflist *bif;
 	struct ifnet *ifp;
 
-	BRIDGE_LOCK_ASSERT(sc);
+	NET_EPOCH_ASSERT();
 
 	CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) {
 		ifp = bif->bif_ifp;
@@ -1004,7 +1000,7 @@ bridge_lookup_member_if(struct bridge_softc *sc, struc
 {
 	struct bridge_iflist *bif;
 
-	BRIDGE_LOCK_ASSERT(sc);
+	NET_EPOCH_ASSERT();
 
 	CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) {
 		if (bif->bif_ifp == member_ifp)
@@ -1014,6 +1010,16 @@ bridge_lookup_member_if(struct bridge_softc *sc, struc
 	return (NULL);
 }
 
+static void
+bridge_delete_member_cb(struct epoch_context *ctx)
+{
+	struct bridge_iflist *bif;
+
+	bif = __containerof(ctx, struct bridge_iflist, bif_epoch_ctx);
+
+	free(bif, M_DEVBUF);
+}
+
 /*
  * bridge_delete_member:
  *
@@ -1033,9 +1039,7 @@ bridge_delete_member(struct bridge_softc *sc, struct b
 		bstp_disable(&bif->bif_stp);
 
 	ifs->if_bridge = NULL;
-	BRIDGE_XLOCK(sc);
 	CK_LIST_REMOVE(bif, bif_next);
-	BRIDGE_XDROP(sc);
 
 	/*
 	 * If removing the interface that gave the bridge its mac address, set
@@ -1094,7 +1098,8 @@ bridge_delete_member(struct bridge_softc *sc, struct b
 	}
 	bstp_destroy(&bif->bif_stp);	/* prepare to free */
 	BRIDGE_LOCK(sc);
-	free(bif, M_DEVBUF);
+
+	NET_EPOCH_CALL(bridge_delete_member_cb, &bif->bif_epoch_ctx);
 }
 
 /*
@@ -1111,7 +1116,8 @@ bridge_delete_span(struct bridge_softc *sc, struct bri
 	    ("%s: not a span interface", __func__));
 
 	CK_LIST_REMOVE(bif, bif_next);
-	free(bif, M_DEVBUF);
+
+	NET_EPOCH_CALL(bridge_delete_member_cb, &bif->bif_epoch_ctx);
 }
 
 static int
@@ -1167,7 +1173,6 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg)
 		 * If any, remove all inet6 addresses from the member
 		 * interfaces.
 		 */
-		BRIDGE_XLOCK(sc);
 		CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) {
  			if (in6ifa_llaonifp(bif->bif_ifp)) {
 				BRIDGE_UNLOCK(sc);
@@ -1180,7 +1185,6 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg)
 				    bif->bif_ifp->if_xname);
 			}
 		}
-		BRIDGE_XDROP(sc);
 		if (in6ifa_llaonifp(ifs)) {
 			BRIDGE_UNLOCK(sc);
 			in6_ifdetach(ifs);
@@ -1494,12 +1498,17 @@ bridge_ioctl_saddr(struct bridge_softc *sc, void *arg)
 	struct bridge_iflist *bif;
 	int error;
 
+	NET_EPOCH_ASSERT();
+
 	bif = bridge_lookup_member(sc, req->ifba_ifsname);
 	if (bif == NULL)
 		return (ENOENT);
 
+	/* bridge_rtupdate() may acquire the lock. */
+	BRIDGE_UNLOCK(sc);
 	error = bridge_rtupdate(sc, req->ifba_dst, req->ifba_vlan, bif, 1,
 	    req->ifba_flags);
+	BRIDGE_LOCK(sc);
 
 	return (error);
 }
@@ -1838,6 +1847,7 @@ bridge_ifdetach(void *arg __unused, struct ifnet *ifp)
 {
 	struct bridge_softc *sc = ifp->if_bridge;
 	struct bridge_iflist *bif;
+	struct epoch_tracker et;
 
 	if (ifp->if_flags & IFF_RENAMING)
 		return;
@@ -1848,6 +1858,7 @@ bridge_ifdetach(void *arg __unused, struct ifnet *ifp)
 		 */
 		return;
 	}
+	NET_EPOCH_ENTER(et);
 	/* Check if the interface is a bridge member */
 	if (sc != NULL) {
 		BRIDGE_LOCK(sc);
@@ -1857,6 +1868,7 @@ bridge_ifdetach(void *arg __unused, struct ifnet *ifp)
 			bridge_delete_member(sc, bif, 1);
 
 		BRIDGE_UNLOCK(sc);
+		NET_EPOCH_EXIT(et);
 		return;
 	}
 
@@ -1873,6 +1885,7 @@ bridge_ifdetach(void *arg __unused, struct ifnet *ifp)
 		BRIDGE_UNLOCK(sc);
 	}
 	BRIDGE_LIST_UNLOCK();
+	NET_EPOCH_EXIT(et);
 }
 
 /*
@@ -1909,6 +1922,7 @@ bridge_stop(struct ifnet *ifp, int disable)
 {
 	struct bridge_softc *sc = ifp->if_softc;
 
+	NET_EPOCH_ASSERT();
 	BRIDGE_LOCK_ASSERT(sc);
 
 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
@@ -2032,6 +2046,8 @@ bridge_output(struct ifnet *ifp, struct mbuf *m, struc
 	struct bridge_softc *sc;
 	uint16_t vlan;
 
+	NET_EPOCH_ASSERT();
+
 	if (m->m_len < ETHER_HDR_LEN) {
 		m = m_pullup(m, ETHER_HDR_LEN);
 		if (m == NULL)
@@ -2042,7 +2058,6 @@ bridge_output(struct ifnet *ifp, struct mbuf *m, struc
 	sc = ifp->if_bridge;
 	vlan = VLANTAGOF(m);
 
-	BRIDGE_LOCK(sc);
 	bifp = sc->sc_ifp;
 
 	/*
@@ -2069,16 +2084,10 @@ bridge_output(struct ifnet *ifp, struct mbuf *m, struc
 	if (dst_if == NULL) {
 		struct bridge_iflist *bif;
 		struct mbuf *mc;
-		int error = 0, used = 0;
+		int used = 0;
 
 		bridge_span(sc, m);
 
-		BRIDGE_LOCK2REF(sc, error);
-		if (error) {
-			m_freem(m);
-			return (0);
-		}
-
 		CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) {
 			dst_if = bif->bif_ifp;
 
@@ -2112,7 +2121,6 @@ bridge_output(struct ifnet *ifp, struct mbuf *m, struc
 		}
 		if (used == 0)
 			m_freem(m);
-		BRIDGE_UNREF(sc);
 		return (0);
 	}
 
@@ -2124,11 +2132,9 @@ sendunicast:
 	bridge_span(sc, m);
 	if ((dst_if->if_drv_flags & IFF_DRV_RUNNING) == 0) {
 		m_freem(m);
-		BRIDGE_UNLOCK(sc);
 		return (0);
 	}
 
-	BRIDGE_UNLOCK(sc);
 	bridge_enqueue(sc, dst_if, m);
 	return (0);
 }
@@ -2153,10 +2159,8 @@ bridge_transmit(struct ifnet *ifp, struct mbuf *m)
 
 	eh = mtod(m, struct ether_header *);
 
-	BRIDGE_LOCK(sc);
 	if (((m->m_flags & (M_BCAST|M_MCAST)) == 0) &&
 	    (dst_if = bridge_rtlookup(sc, eh->ether_dhost, 1)) != NULL) {
-		BRIDGE_UNLOCK(sc);
 		error = bridge_enqueue(sc, dst_if, m);
 	} else
 		bridge_broadcast(sc, ifp, m, 0);
@@ -2190,6 +2194,8 @@ bridge_forward(struct bridge_softc *sc, struct bridge_
 	uint8_t *dst;
 	int error;
 
+	NET_EPOCH_ASSERT();
+
 	src_if = m->m_pkthdr.rcvif;
 	ifp = sc->sc_ifp;
 
@@ -2268,12 +2274,10 @@ bridge_forward(struct bridge_softc *sc, struct bridge_
 	    || PFIL_HOOKED_IN(V_inet6_pfil_head)
 #endif
 	    ) {
-		BRIDGE_UNLOCK(sc);
 		if (bridge_pfil(&m, ifp, src_if, PFIL_IN) != 0)
 			return;
 		if (m == NULL)
 			return;
-		BRIDGE_LOCK(sc);
 	}
 
 	if (dst_if == NULL) {
@@ -2301,8 +2305,6 @@ bridge_forward(struct bridge_softc *sc, struct bridge_
 	    dbif->bif_stp.bp_state == BSTP_IFSTATE_DISCARDING)
 		goto drop;
 
-	BRIDGE_UNLOCK(sc);
-
 	if (PFIL_HOOKED_OUT(V_inet_pfil_head)
 #ifdef INET6
 	    || PFIL_HOOKED_OUT(V_inet6_pfil_head)
@@ -2318,7 +2320,6 @@ bridge_forward(struct bridge_softc *sc, struct bridge_
 	return;
 
 drop:
-	BRIDGE_UNLOCK(sc);
 	m_freem(m);
 }
 
@@ -2339,6 +2340,8 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
 	uint16_t vlan;
 	int error;
 
+	NET_EPOCH_ASSERT();
+
 	if ((sc->sc_ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 		return (m);
 
@@ -2359,10 +2362,8 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
 		m_freem(m);
 		return (NULL);
 	}
-	BRIDGE_LOCK(sc);
 	bif = bridge_lookup_member_if(sc, ifp);
 	if (bif == NULL) {
-		BRIDGE_UNLOCK(sc);
 		return (m);
 	}
 
@@ -2375,13 +2376,11 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
 		if (memcmp(eh->ether_dhost, bstp_etheraddr,
 		    ETHER_ADDR_LEN) == 0) {
 			bstp_input(&bif->bif_stp, ifp, m); /* consumes mbuf */
-			BRIDGE_UNLOCK(sc);
 			return (NULL);
 		}
 
 		if ((bif->bif_flags & IFBIF_STP) &&
 		    bif->bif_stp.bp_state == BSTP_IFSTATE_DISCARDING) {
-			BRIDGE_UNLOCK(sc);
 			return (m);
 		}
 
@@ -2392,7 +2391,6 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
 		 */
 		mc = m_dup(m, M_NOWAIT);
 		if (mc == NULL) {
-			BRIDGE_UNLOCK(sc);
 			return (m);
 		}
 
@@ -2424,7 +2422,6 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
 
 	if ((bif->bif_flags & IFBIF_STP) &&
 	    bif->bif_stp.bp_state == BSTP_IFSTATE_DISCARDING) {
-		BRIDGE_UNLOCK(sc);
 		return (m);
 	}
 
@@ -2458,7 +2455,6 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
 			error = bridge_rtupdate(sc, eh->ether_shost,	\
 			    vlan, bif, 0, IFBAF_DYNAMIC);		\
 			if (error && bif->bif_addrmax) {		\
-				BRIDGE_UNLOCK(sc);			\
 				m_freem(m);				\
 				return (NULL);				\
 			}						\
@@ -2466,7 +2462,6 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
 		m->m_pkthdr.rcvif = iface;				\
 		if ((iface) == ifp) {					\
 			/* Skip bridge processing... src == dest */	\
-			BRIDGE_UNLOCK(sc);				\
 			return (m);					\
 		}							\
 		/* It's passing over or to the bridge, locally. */	\
@@ -2478,13 +2473,11 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
 		     OR_PFIL_HOOKED_INET6)) {				\
 			if (bridge_pfil(&m, NULL, ifp,			\
 			    PFIL_IN) != 0 || m == NULL) {		\
-				BRIDGE_UNLOCK(sc);			\
 				return (NULL);				\
 			}						\
 		}							\
 		if ((iface) != bifp)					\
 			ETHER_BPF_MTAP(iface, m);			\
-		BRIDGE_UNLOCK(sc);					\
 		return (m);						\
 	}								\
 									\
@@ -2492,7 +2485,6 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
 	if (memcmp(IF_LLADDR((iface)), eh->ether_shost, ETHER_ADDR_LEN) == 0 \
 	    OR_CARP_CHECK_WE_ARE_SRC((iface))			\
 	    ) {								\
-		BRIDGE_UNLOCK(sc);					\
 		m_freem(m);						\
 		return (NULL);						\
 	}
@@ -2543,16 +2535,12 @@ bridge_broadcast(struct bridge_softc *sc, struct ifnet
 	struct bridge_iflist *dbif, *sbif;
 	struct mbuf *mc;
 	struct ifnet *dst_if;
-	int error = 0, used = 0, i;
+	int used = 0, i;
 
+	NET_EPOCH_ASSERT();
+
 	sbif = bridge_lookup_member_if(sc, src_if);
 
-	BRIDGE_LOCK2REF(sc, error);
-	if (error) {
-		m_freem(m);
-		return;
-	}
-
 	/* Filter on the bridge interface before broadcasting */
 	if (runfilt && (PFIL_HOOKED_OUT(V_inet_pfil_head)
 #ifdef INET6
@@ -2560,9 +2548,9 @@ bridge_broadcast(struct bridge_softc *sc, struct ifnet
 #endif
 	    )) {
 		if (bridge_pfil(&m, sc->sc_ifp, NULL, PFIL_OUT) != 0)
-			goto out;
+			return;
 		if (m == NULL)
-			goto out;
+			return;
 	}
 
 	CK_LIST_FOREACH(dbif, &sc->sc_iflist, bif_next) {
@@ -2625,9 +2613,6 @@ bridge_broadcast(struct bridge_softc *sc, struct ifnet
 	}
 	if (used == 0)
 		m_freem(m);
-
-out:
-	BRIDGE_UNREF(sc);
 }
 
 /*
@@ -2643,6 +2628,8 @@ bridge_span(struct bridge_softc *sc, struct mbuf *m)
 	struct ifnet *dst_if;
 	struct mbuf *mc;
 
+	NET_EPOCH_ASSERT();
+
 	if (CK_LIST_EMPTY(&sc->sc_spanlist))
 		return;
 
@@ -2674,7 +2661,8 @@ bridge_rtupdate(struct bridge_softc *sc, const uint8_t
 	struct bridge_rtnode *brt;
 	int error;
 
-	BRIDGE_LOCK_ASSERT(sc);
+	NET_EPOCH_ASSERT();
+	BRIDGE_UNLOCK_ASSERT(sc);
 
 	/* Check the source address is valid and not multicast. */
 	if (ETHER_IS_MULTICAST(dst) ||
@@ -2691,13 +2679,24 @@ bridge_rtupdate(struct bridge_softc *sc, const uint8_t
 	 * update it, otherwise create a new one.
 	 */
 	if ((brt = bridge_rtnode_lookup(sc, dst, vlan)) == NULL) {
+		BRIDGE_LOCK(sc);
+
+		/* Check again, now that we have the lock. There could have
+		 * been a race and we only want to insert this once. */
+		if ((brt = bridge_rtnode_lookup(sc, dst, vlan)) != NULL) {
+			BRIDGE_UNLOCK(sc);
+			return (0);
+		}
+
 		if (sc->sc_brtcnt >= sc->sc_brtmax) {
 			sc->sc_brtexceeded++;
+			BRIDGE_UNLOCK(sc);
 			return (ENOSPC);
 		}
 		/* Check per interface address limits (if enabled) */
 		if (bif->bif_addrmax && bif->bif_addrcnt >= bif->bif_addrmax) {
 			bif->bif_addrexceeded++;
+			BRIDGE_UNLOCK(sc);
 			return (ENOSPC);
 		}
 
@@ -2707,8 +2706,11 @@ bridge_rtupdate(struct bridge_softc *sc, const uint8_t
 		 * address.
 		 */
 		brt = uma_zalloc(V_bridge_rtnode_zone, M_NOWAIT | M_ZERO);
-		if (brt == NULL)
+		if (brt == NULL) {
+			BRIDGE_UNLOCK(sc);
 			return (ENOMEM);
+		}
+		brt->brt_vnet = curvnet;
 
 		if (bif->bif_flags & IFBIF_STICKY)
 			brt->brt_flags = IFBAF_STICKY;
@@ -2720,17 +2722,22 @@ bridge_rtupdate(struct bridge_softc *sc, const uint8_t
 
 		if ((error = bridge_rtnode_insert(sc, brt)) != 0) {
 			uma_zfree(V_bridge_rtnode_zone, brt);
+			BRIDGE_UNLOCK(sc);
 			return (error);
 		}
 		brt->brt_dst = bif;
 		bif->bif_addrcnt++;
+
+		BRIDGE_UNLOCK(sc);
 	}
 
 	if ((brt->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC &&
 	    brt->brt_dst != bif) {
+		BRIDGE_LOCK(sc);
 		brt->brt_dst->bif_addrcnt--;
 		brt->brt_dst = bif;
 		brt->brt_dst->bif_addrcnt++;
+		BRIDGE_UNLOCK(sc);
 	}
 
 	if ((flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC)
@@ -2751,7 +2758,7 @@ bridge_rtlookup(struct bridge_softc *sc, const uint8_t
 {
 	struct bridge_rtnode *brt;
 
-	BRIDGE_LOCK_ASSERT(sc);
+	NET_EPOCH_ASSERT();
 
 	if ((brt = bridge_rtnode_lookup(sc, addr, vlan)) == NULL)
 		return (NULL);
@@ -2771,6 +2778,7 @@ bridge_rttrim(struct bridge_softc *sc)
 {
 	struct bridge_rtnode *brt, *nbrt;
 
+	NET_EPOCH_ASSERT();
 	BRIDGE_LOCK_ASSERT(sc);
 
 	/* Make sure we actually need to do this. */
@@ -2800,7 +2808,9 @@ static void
 bridge_timer(void *arg)
 {
 	struct bridge_softc *sc = arg;
+	struct epoch_tracker et;
 
+	NET_EPOCH_ENTER(et);
 	BRIDGE_LOCK_ASSERT(sc);
 
 	/* Destruction of rtnodes requires a proper vnet context */
@@ -2811,6 +2821,7 @@ bridge_timer(void *arg)
 		callout_reset(&sc->sc_brcallout,
 		    bridge_rtable_prune_period * hz, bridge_timer, sc);
 	CURVNET_RESTORE();
+	NET_EPOCH_EXIT(et);
 }
 
 /*
@@ -2823,6 +2834,7 @@ bridge_rtage(struct bridge_softc *sc)
 {
 	struct bridge_rtnode *brt, *nbrt;
 
+	NET_EPOCH_ASSERT();
 	BRIDGE_LOCK_ASSERT(sc);
 
 	CK_LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) {
@@ -2843,6 +2855,7 @@ bridge_rtflush(struct bridge_softc *sc, int full)
 {
 	struct bridge_rtnode *brt, *nbrt;
 
+	NET_EPOCH_ASSERT();
 	BRIDGE_LOCK_ASSERT(sc);
 
 	CK_LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) {
@@ -2862,6 +2875,7 @@ bridge_rtdaddr(struct bridge_softc *sc, const uint8_t 
 	struct bridge_rtnode *brt;
 	int found = 0;
 
+	NET_EPOCH_ASSERT();
 	BRIDGE_LOCK_ASSERT(sc);
 
 	/*
@@ -2886,6 +2900,7 @@ bridge_rtdelete(struct bridge_softc *sc, struct ifnet 
 {
 	struct bridge_rtnode *brt, *nbrt;
 
+	NET_EPOCH_ASSERT();
 	BRIDGE_LOCK_ASSERT(sc);
 
 	CK_LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) {
@@ -2990,7 +3005,7 @@ bridge_rtnode_lookup(struct bridge_softc *sc, const ui
 	uint32_t hash;
 	int dir;
 
-	BRIDGE_LOCK_ASSERT(sc);
+	NET_EPOCH_ASSERT();
 
 	hash = bridge_rthash(sc, addr);
 	CK_LIST_FOREACH(brt, &sc->sc_rthash[hash], brt_hash) {
@@ -3053,6 +3068,18 @@ out:
 	return (0);
 }
 
+static void
+bridge_rtnode_destroy_cb(struct epoch_context *ctx)
+{
+	struct bridge_rtnode *brt;
+
+	brt = __containerof(ctx, struct bridge_rtnode, brt_epoch_ctx);
+
+	CURVNET_SET(brt->brt_vnet);
+	uma_zfree(V_bridge_rtnode_zone, brt);
+	CURVNET_RESTORE();
+}
+
 /*
  * bridge_rtnode_destroy:
  *
@@ -3061,6 +3088,7 @@ out:
 static void
 bridge_rtnode_destroy(struct bridge_softc *sc, struct bridge_rtnode *brt)
 {
+	NET_EPOCH_ASSERT();
 	BRIDGE_LOCK_ASSERT(sc);
 
 	CK_LIST_REMOVE(brt, brt_hash);
@@ -3068,7 +3096,8 @@ bridge_rtnode_destroy(struct bridge_softc *sc, struct 
 	CK_LIST_REMOVE(brt, brt_list);
 	sc->sc_brtcnt--;
 	brt->brt_dst->bif_addrcnt--;
-	uma_zfree(V_bridge_rtnode_zone, brt);
+
+	NET_EPOCH_CALL(bridge_rtnode_destroy_cb, &brt->brt_epoch_ctx);
 }
 
 /*
@@ -3081,7 +3110,9 @@ bridge_rtable_expire(struct ifnet *ifp, int age)
 {
 	struct bridge_softc *sc = ifp->if_bridge;
 	struct bridge_rtnode *brt;
+	struct epoch_tracker et;
 
+	NET_EPOCH_ENTER(et);
 	CURVNET_SET(ifp->if_vnet);
 	BRIDGE_LOCK(sc);
 
@@ -3102,6 +3133,7 @@ bridge_rtable_expire(struct ifnet *ifp, int age)
 	}
 	BRIDGE_UNLOCK(sc);
 	CURVNET_RESTORE();
+	NET_EPOCH_EXIT(et);
 }
 
 /*
@@ -3607,17 +3639,20 @@ bridge_linkstate(struct ifnet *ifp)
 {
 	struct bridge_softc *sc = ifp->if_bridge;
 	struct bridge_iflist *bif;
+	struct epoch_tracker et;
 
-	BRIDGE_LOCK(sc);
+	NET_EPOCH_ENTER(et);
+
 	bif = bridge_lookup_member_if(sc, ifp);
 	if (bif == NULL) {
-		BRIDGE_UNLOCK(sc);
+		NET_EPOCH_EXIT(et);
 		return;
 	}
 	bridge_linkcheck(sc);
-	BRIDGE_UNLOCK(sc);
 
 	bstp_linkstate(&bif->bif_stp);
+
+	NET_EPOCH_EXIT(et);
 }
 
 static void
@@ -3626,7 +3661,8 @@ bridge_linkcheck(struct bridge_softc *sc)
 	struct bridge_iflist *bif;
 	int new_link, hasls;
 
-	BRIDGE_LOCK_ASSERT(sc);
+	NET_EPOCH_ASSERT();
+
 	new_link = LINK_STATE_DOWN;
 	hasls = 0;
 	/* Our link is considered up if at least one of our ports is active */



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202004261622.03QGMZWt001811>