Date: Mon, 7 Oct 2019 22:40:06 +0000 (UTC) From: Gleb Smirnoff <glebius@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r353292 - in head/sys: contrib/ipfilter/netinet dev/firewire dev/iicbus dev/usb/net kern net netgraph netinet netinet6 netipsec netpfil/ipfw netpfil/pf ofed/drivers/infiniband/ulp/ipoib Message-ID: <201910072240.x97Me60x065650@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: glebius Date: Mon Oct 7 22:40:05 2019 New Revision: 353292 URL: https://svnweb.freebsd.org/changeset/base/353292 Log: Widen NET_EPOCH coverage. When epoch(9) was introduced to network stack, it was basically dropped in place of existing locking, which was mutexes and rwlocks. For the sake of performance mutex covered areas were as small as possible, so became epoch covered areas. However, epoch doesn't introduce any contention, it just delays memory reclaim. So, there is no point to minimise epoch covered areas in sense of performance. Meanwhile entering/exiting epoch also has non-zero CPU usage, so doing this less often is a win. Not the least is also code maintainability. In the new paradigm we can assume that at any stage of processing a packet, we are inside network epoch. This makes coding both input and output path way easier. On output path we already enter epoch quite early - in the ip_output(), in the ip6_output(). This patch does the same for the input path. All ISR processing, network related callouts, other ways of packet injection to the network stack shall be performed in net_epoch. Any leaf function that walks network configuration now asserts epoch. Tricky part is configuration code paths - ioctls, sysctls. They also call into leaf functions, so some need to be changed. This patch would introduce more epoch recursions (see EPOCH_TRACE) than we had before. They will be cleaned up separately, as several of them aren't trivial. Note, that unlike a lock recursion the epoch recursion is safe and just wastes a bit of resources. Reviewed by: gallatin, hselasky, cy, adrian, kristof Differential Revision: https://reviews.freebsd.org/D19111 Modified: head/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c head/sys/dev/firewire/if_fwip.c head/sys/dev/iicbus/if_ic.c head/sys/dev/usb/net/if_usie.c head/sys/dev/usb/net/uhso.c head/sys/dev/usb/net/usb_ethernet.c head/sys/kern/uipc_socket.c head/sys/net/if.c head/sys/net/if_ethersubr.c head/sys/net/if_gif.c head/sys/net/if_me.c head/sys/net/if_stf.c head/sys/net/if_tuntap.c head/sys/net/if_vlan.c head/sys/net/netisr.c head/sys/net/route.c head/sys/net/rtsock.c head/sys/netgraph/ng_ether.c head/sys/netgraph/ng_iface.c head/sys/netgraph/ng_ip_input.c head/sys/netinet/if_ether.c head/sys/netinet/igmp.c head/sys/netinet/in.c head/sys/netinet/in_mcast.c head/sys/netinet/in_rmx.c head/sys/netinet/in_var.h head/sys/netinet/ip_carp.c head/sys/netinet/ip_encap.c head/sys/netinet/ip_icmp.c head/sys/netinet/ip_input.c head/sys/netinet/ip_mroute.c head/sys/netinet/ip_options.c head/sys/netinet/ip_output.c head/sys/netinet/sctp_bsd_addr.c head/sys/netinet6/icmp6.c head/sys/netinet6/in6.c head/sys/netinet6/in6_ifattach.c head/sys/netinet6/in6_mcast.c head/sys/netinet6/in6_var.h head/sys/netinet6/ip6_output.c head/sys/netinet6/mld6.c head/sys/netinet6/nd6.c head/sys/netinet6/nd6_nbr.c head/sys/netinet6/nd6_rtr.c head/sys/netinet6/raw_ip6.c head/sys/netipsec/xform_ipcomp.c head/sys/netpfil/ipfw/ip_dn_io.c head/sys/netpfil/pf/pf_if.c head/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c Modified: head/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c ============================================================================== --- head/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c Mon Oct 7 20:41:55 2019 (r353291) +++ head/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c Mon Oct 7 22:40:05 2019 (r353292) @@ -1312,8 +1312,10 @@ ipf_inject(fin, m) fr_info_t *fin; mb_t *m; { + struct epoch_tracker et; int error = 0; + NET_EPOCH_ENTER(et); if (fin->fin_out == 0) { netisr_dispatch(NETISR_IP, m); } else { @@ -1321,6 +1323,7 @@ ipf_inject(fin, m) fin->fin_ip->ip_off = ntohs(fin->fin_ip->ip_off); error = ip_output(m, NULL, NULL, IP_FORWARDING, NULL, NULL); } + NET_EPOCH_EXIT(et); return error; } Modified: head/sys/dev/firewire/if_fwip.c ============================================================================== --- head/sys/dev/firewire/if_fwip.c Mon Oct 7 20:41:55 2019 (r353291) +++ head/sys/dev/firewire/if_fwip.c Mon Oct 7 22:40:05 2019 (r353292) @@ -708,6 +708,7 @@ fwip_start_send (void *arg, int count) static void fwip_stream_input(struct fw_xferq *xferq) { + struct epoch_tracker et; struct mbuf *m, *m0; struct m_tag *mtag; struct ifnet *ifp; @@ -720,6 +721,7 @@ fwip_stream_input(struct fw_xferq *xferq) fwip = (struct fwip_softc *)xferq->sc; ifp = fwip->fw_softc.fwip_ifp; + NET_EPOCH_ENTER(et); while ((sxfer = STAILQ_FIRST(&xferq->stvalid)) != NULL) { STAILQ_REMOVE_HEAD(&xferq->stvalid, link); fp = mtod(sxfer->mbuf, struct fw_pkt *); @@ -808,6 +810,7 @@ fwip_stream_input(struct fw_xferq *xferq) firewire_input(ifp, m, src); if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); } + NET_EPOCH_EXIT(et); if (STAILQ_FIRST(&xferq->stfree) != NULL) fwip->fd.fc->irx_enable(fwip->fd.fc, fwip->dma_ch); } Modified: head/sys/dev/iicbus/if_ic.c ============================================================================== --- head/sys/dev/iicbus/if_ic.c Mon Oct 7 20:41:55 2019 (r353291) +++ head/sys/dev/iicbus/if_ic.c Mon Oct 7 22:40:05 2019 (r353292) @@ -309,9 +309,13 @@ icintr(device_t dev, int event, char *ptr) BPF_TAP(sc->ic_ifp, sc->ic_ifbuf, len + ICHDRLEN); top = m_devget(sc->ic_ifbuf + ICHDRLEN, len, 0, sc->ic_ifp, 0); if (top) { + struct epoch_tracker et; + mtx_unlock(&sc->ic_lock); M_SETFIB(top, sc->ic_ifp->if_fib); + NET_EPOCH_ENTER(et); netisr_dispatch(NETISR_IP, top); + NET_EPOCH_EXIT(et); mtx_lock(&sc->ic_lock); } break; Modified: head/sys/dev/usb/net/if_usie.c ============================================================================== --- head/sys/dev/usb/net/if_usie.c Mon Oct 7 20:41:55 2019 (r353291) +++ head/sys/dev/usb/net/if_usie.c Mon Oct 7 22:40:05 2019 (r353292) @@ -773,6 +773,7 @@ tr_setup: static void usie_if_rx_callback(struct usb_xfer *xfer, usb_error_t error) { + struct epoch_tracker et; struct usie_softc *sc = usbd_xfer_softc(xfer); struct ifnet *ifp = sc->sc_ifp; struct mbuf *m0; @@ -852,6 +853,7 @@ tr_setup: err = pkt = 0; /* HW can aggregate multiple frames in a single USB xfer */ + NET_EPOCH_ENTER(et); for (;;) { rxd = mtod(m, struct usie_desc *); @@ -918,6 +920,7 @@ tr_setup: m->m_data += diff; m->m_pkthdr.len = (m->m_len -= diff); } + NET_EPOCH_EXIT(et); mtx_lock(&sc->sc_mtx); Modified: head/sys/dev/usb/net/uhso.c ============================================================================== --- head/sys/dev/usb/net/uhso.c Mon Oct 7 20:41:55 2019 (r353291) +++ head/sys/dev/usb/net/uhso.c Mon Oct 7 22:40:05 2019 (r353292) @@ -1664,6 +1664,7 @@ tr_setup: static void uhso_if_rxflush(void *arg) { + struct epoch_tracker et; struct uhso_softc *sc = arg; struct ifnet *ifp = sc->sc_ifp; uint8_t *cp; @@ -1677,6 +1678,7 @@ uhso_if_rxflush(void *arg) m = NULL; mwait = sc->sc_mwait; + NET_EPOCH_ENTER(et); for (;;) { if (m == NULL) { if ((m = mbufq_dequeue(&sc->sc_rxq)) == NULL) @@ -1787,6 +1789,7 @@ uhso_if_rxflush(void *arg) m = m0 != NULL ? m0 : NULL; mtx_lock(&sc->sc_mtx); } + NET_EPOCH_EXIT(et); sc->sc_mwait = mwait; } Modified: head/sys/dev/usb/net/usb_ethernet.c ============================================================================== --- head/sys/dev/usb/net/usb_ethernet.c Mon Oct 7 20:41:55 2019 (r353291) +++ head/sys/dev/usb/net/usb_ethernet.c Mon Oct 7 22:40:05 2019 (r353292) @@ -645,22 +645,21 @@ void uether_rxflush(struct usb_ether *ue) { struct ifnet *ifp = ue->ue_ifp; - struct mbuf *m; + struct epoch_tracker et; + struct mbuf *m, *n; UE_LOCK_ASSERT(ue, MA_OWNED); - for (;;) { - m = mbufq_dequeue(&ue->ue_rxq); - if (m == NULL) - break; - - /* - * The USB xfer has been resubmitted so its safe to unlock now. - */ - UE_UNLOCK(ue); + n = mbufq_flush(&ue->ue_rxq); + UE_UNLOCK(ue); + NET_EPOCH_ENTER(et); + while ((m = n) != NULL) { + n = STAILQ_NEXT(m, m_stailqpkt); + m->m_nextpkt = NULL; ifp->if_input(ifp, m); - UE_LOCK(ue); } + NET_EPOCH_EXIT(et); + UE_LOCK(ue); } /* Modified: head/sys/kern/uipc_socket.c ============================================================================== --- head/sys/kern/uipc_socket.c Mon Oct 7 20:41:55 2019 (r353291) +++ head/sys/kern/uipc_socket.c Mon Oct 7 22:40:05 2019 (r353292) @@ -146,6 +146,8 @@ __FBSDID("$FreeBSD$"); #include <netinet/tcp.h> #include <net/vnet.h> +#include <net/if.h> /* XXXGL: net_epoch should move out there */ +#include <net/if_var.h> /* XXXGL: net_epoch should move out there */ #include <security/mac/mac_framework.h> Modified: head/sys/net/if.c ============================================================================== --- head/sys/net/if.c Mon Oct 7 20:41:55 2019 (r353291) +++ head/sys/net/if.c Mon Oct 7 22:40:05 2019 (r353292) @@ -351,17 +351,14 @@ ifnet_byindex(u_short idx) struct ifnet * ifnet_byindex_ref(u_short idx) { - struct epoch_tracker et; struct ifnet *ifp; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + ifp = ifnet_byindex_locked(idx); - if (ifp == NULL || (ifp->if_flags & IFF_DYING)) { - NET_EPOCH_EXIT(et); + if (ifp == NULL || (ifp->if_flags & IFF_DYING)) return (NULL); - } if_ref(ifp); - NET_EPOCH_EXIT(et); return (ifp); } @@ -425,15 +422,14 @@ ifnet_setbyindex(u_short idx, struct ifnet *ifp) struct ifaddr * ifaddr_byindex(u_short idx) { - struct epoch_tracker et; struct ifnet *ifp; struct ifaddr *ifa = NULL; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + ifp = ifnet_byindex_locked(idx); if (ifp != NULL && (ifa = ifp->if_addr) != NULL) ifa_ref(ifa); - NET_EPOCH_EXIT(et); return (ifa); } @@ -1640,39 +1636,32 @@ ifgr_groups_get(void *ifgrp) static int if_getgroup(struct ifgroupreq *ifgr, struct ifnet *ifp) { - struct epoch_tracker et; int len, error; struct ifg_list *ifgl; struct ifg_req ifgrq, *ifgp; + NET_EPOCH_ASSERT(); + if (ifgr->ifgr_len == 0) { - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) ifgr->ifgr_len += sizeof(struct ifg_req); - NET_EPOCH_EXIT(et); return (0); } len = ifgr->ifgr_len; ifgp = ifgr_groups_get(ifgr); /* XXX: wire */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) { - if (len < sizeof(ifgrq)) { - NET_EPOCH_EXIT(et); + if (len < sizeof(ifgrq)) return (EINVAL); - } bzero(&ifgrq, sizeof ifgrq); strlcpy(ifgrq.ifgrq_group, ifgl->ifgl_group->ifg_group, sizeof(ifgrq.ifgrq_group)); - if ((error = copyout(&ifgrq, ifgp, sizeof(struct ifg_req)))) { - NET_EPOCH_EXIT(et); + if ((error = copyout(&ifgrq, ifgp, sizeof(struct ifg_req)))) return (error); - } len -= sizeof(ifgrq); ifgp++; } - NET_EPOCH_EXIT(et); return (0); } @@ -1972,7 +1961,8 @@ ifa_ifwithaddr(const struct sockaddr *addr) struct ifnet *ifp; struct ifaddr *ifa; - MPASS(in_epoch(net_epoch_preempt)); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != addr->sa_family) @@ -2324,17 +2314,22 @@ if_link_state_change(struct ifnet *ifp, int link_state ifp->if_link_state = link_state; + /* XXXGL: reference ifp? */ taskqueue_enqueue(taskqueue_swi, &ifp->if_linktask); } static void do_link_state_change(void *arg, int pending) { - struct ifnet *ifp = (struct ifnet *)arg; - int link_state = ifp->if_link_state; - CURVNET_SET(ifp->if_vnet); + struct epoch_tracker et; + struct ifnet *ifp; + int link_state; - /* Notify that the link state has changed. */ + NET_EPOCH_ENTER(et); + ifp = arg; + link_state = ifp->if_link_state; + + CURVNET_SET(ifp->if_vnet); rt_ifmsg(ifp); if (ifp->if_vlantrunk != NULL) (*vlan_link_state_p)(ifp); @@ -2360,6 +2355,7 @@ do_link_state_change(void *arg, int pending) (link_state == LINK_STATE_UP) ? "UP" : "DOWN" ); EVENTHANDLER_INVOKE(ifnet_link_event, ifp, link_state); CURVNET_RESTORE(); + NET_EPOCH_EXIT(et); } /* @@ -2912,9 +2908,14 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, break; case CASE_IOC_IFGROUPREQ(SIOCGIFGROUP): - if ((error = if_getgroup((struct ifgroupreq *)data, ifp))) - return (error); + { + struct epoch_tracker et; + + NET_EPOCH_ENTER(et); + error = if_getgroup((struct ifgroupreq *)data, ifp); + NET_EPOCH_EXIT(et); break; + } case CASE_IOC_IFGROUPREQ(SIOCDIFGROUP): error = priv_check(td, PRIV_NET_DELIFGROUP); @@ -2983,7 +2984,7 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, s #ifdef COMPAT_FREEBSD32 caddr_t saved_data = NULL; struct ifmediareq ifmr; - struct ifmediareq *ifmrp; + struct ifmediareq *ifmrp = NULL; #endif struct ifnet *ifp; struct ifreq *ifr; @@ -2999,12 +3000,10 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, s } #endif - switch (cmd) { case SIOCGIFCONF: error = ifconf(cmd, data); - CURVNET_RESTORE(); - return (error); + goto out_noref; #ifdef COMPAT_FREEBSD32 case SIOCGIFCONF32: @@ -3017,16 +3016,14 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, s ifc.ifc_buf = PTRIN(ifc32->ifc_buf); error = ifconf(SIOCGIFCONF, (void *)&ifc); - CURVNET_RESTORE(); if (error == 0) ifc32->ifc_len = ifc.ifc_len; - return (error); + goto out_noref; } #endif } #ifdef COMPAT_FREEBSD32 - ifmrp = NULL; switch (cmd) { case SIOCGIFMEDIA32: case SIOCGIFXMEDIA32: @@ -3618,16 +3615,15 @@ if_delmulti(struct ifnet *ifp, struct sockaddr *sa) struct ifmultiaddr *ifma; int lastref; #ifdef INVARIANTS - struct epoch_tracker et; struct ifnet *oifp; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link) if (ifp == oifp) break; if (ifp != oifp) ifp = NULL; - NET_EPOCH_EXIT(et); KASSERT(ifp != NULL, ("%s: ifnet went away", __func__)); #endif @@ -3693,16 +3689,15 @@ if_delmulti_ifma_flags(struct ifmultiaddr *ifma, int f if (ifp == NULL) { printf("%s: ifma_ifp seems to be detached\n", __func__); } else { - struct epoch_tracker et; struct ifnet *oifp; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link) if (ifp == oifp) break; if (ifp != oifp) ifp = NULL; - NET_EPOCH_EXIT(et); } #endif /* @@ -3826,11 +3821,11 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, struct sockaddr_dl *sdl; struct ifaddr *ifa; struct ifreq ifr; - struct epoch_tracker et; int rc; + NET_EPOCH_ASSERT(); + rc = 0; - NET_EPOCH_ENTER(et); ifa = ifp->if_addr; if (ifa == NULL) { rc = EINVAL; @@ -3864,7 +3859,6 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, * to re-init it in order to reprogram its * address filter. */ - NET_EPOCH_EXIT(et); if ((ifp->if_flags & IFF_UP) != 0) { if (ifp->if_ioctl) { ifp->if_flags &= ~IFF_UP; @@ -3879,8 +3873,7 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, } EVENTHANDLER_INVOKE(iflladdr_event, ifp); return (0); - out: - NET_EPOCH_EXIT(et); +out: return (rc); } Modified: head/sys/net/if_ethersubr.c ============================================================================== --- head/sys/net/if_ethersubr.c Mon Oct 7 20:41:55 2019 (r353291) +++ head/sys/net/if_ethersubr.c Mon Oct 7 22:40:05 2019 (r353292) @@ -800,6 +800,7 @@ VNET_SYSUNINIT(vnet_ether_uninit, SI_SUB_PROTO_IF, SI_ static void ether_input(struct ifnet *ifp, struct mbuf *m) { + struct epoch_tracker et; struct mbuf *mn; /* @@ -807,22 +808,24 @@ ether_input(struct ifnet *ifp, struct mbuf *m) * m_nextpkt. We split them up into separate packets here and pass * them up. This allows the drivers to amortize the receive lock. */ + CURVNET_SET_QUIET(ifp->if_vnet); + NET_EPOCH_ENTER(et); while (m) { mn = m->m_nextpkt; m->m_nextpkt = NULL; /* - * We will rely on rcvif being set properly in the deferred context, - * so assert it is correct here. + * We will rely on rcvif being set properly in the deferred + * context, so assert it is correct here. */ MPASS((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0); KASSERT(m->m_pkthdr.rcvif == ifp, ("%s: ifnet mismatch m %p " "rcvif %p ifp %p", __func__, m, m->m_pkthdr.rcvif, ifp)); - CURVNET_SET_QUIET(ifp->if_vnet); netisr_dispatch(NETISR_ETHER, m); - CURVNET_RESTORE(); m = mn; } + NET_EPOCH_EXIT(et); + CURVNET_RESTORE(); } /* @@ -835,6 +838,7 @@ ether_demux(struct ifnet *ifp, struct mbuf *m) int i, isr; u_short ether_type; + NET_EPOCH_ASSERT(); KASSERT(ifp != NULL, ("%s: NULL interface pointer", __func__)); /* Do not grab PROMISC frames in case we are re-entered. */ Modified: head/sys/net/if_gif.c ============================================================================== --- head/sys/net/if_gif.c Mon Oct 7 20:41:55 2019 (r353291) +++ head/sys/net/if_gif.c Mon Oct 7 22:40:05 2019 (r353292) @@ -415,6 +415,8 @@ gif_input(struct mbuf *m, struct ifnet *ifp, int proto struct ifnet *oldifp; int isr, n, af; + NET_EPOCH_ASSERT(); + if (ifp == NULL) { /* just in case */ m_freem(m); Modified: head/sys/net/if_me.c ============================================================================== --- head/sys/net/if_me.c Mon Oct 7 20:41:55 2019 (r353291) +++ head/sys/net/if_me.c Mon Oct 7 22:40:05 2019 (r353292) @@ -451,6 +451,8 @@ me_input(struct mbuf *m, int off, int proto, void *arg struct ip *ip; int hlen; + NET_EPOCH_ASSERT(); + ifp = ME2IFP(sc); /* checks for short packets */ hlen = sizeof(struct mobhdr); Modified: head/sys/net/if_stf.c ============================================================================== --- head/sys/net/if_stf.c Mon Oct 7 20:41:55 2019 (r353291) +++ head/sys/net/if_stf.c Mon Oct 7 22:40:05 2019 (r353292) @@ -613,6 +613,8 @@ in_stf_input(struct mbuf *m, int off, int proto, void u_int8_t otos, itos; struct ifnet *ifp; + NET_EPOCH_ASSERT(); + if (proto != IPPROTO_IPV6) { m_freem(m); return (IPPROTO_DONE); Modified: head/sys/net/if_tuntap.c ============================================================================== --- head/sys/net/if_tuntap.c Mon Oct 7 20:41:55 2019 (r353291) +++ head/sys/net/if_tuntap.c Mon Oct 7 22:40:05 2019 (r353292) @@ -1662,6 +1662,7 @@ tunwrite_l2(struct tuntap_softc *tp, struct mbuf *m) static int tunwrite_l3(struct tuntap_softc *tp, struct mbuf *m) { + struct epoch_tracker et; struct ifnet *ifp; int family, isr; @@ -1702,7 +1703,9 @@ tunwrite_l3(struct tuntap_softc *tp, struct mbuf *m) if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); CURVNET_SET(ifp->if_vnet); M_SETFIB(m, ifp->if_fib); + NET_EPOCH_ENTER(et); netisr_dispatch(isr, m); + NET_EPOCH_EXIT(et); CURVNET_RESTORE(); return (0); } Modified: head/sys/net/if_vlan.c ============================================================================== --- head/sys/net/if_vlan.c Mon Oct 7 20:41:55 2019 (r353291) +++ head/sys/net/if_vlan.c Mon Oct 7 22:40:05 2019 (r353292) @@ -255,7 +255,6 @@ static struct sx _VLAN_SX_ID; #define TRUNK_LOCK_DESTROY(trunk) mtx_destroy(&(trunk)->lock) #define TRUNK_WLOCK(trunk) mtx_lock(&(trunk)->lock) #define TRUNK_WUNLOCK(trunk) mtx_unlock(&(trunk)->lock) -#define TRUNK_LOCK_ASSERT(trunk) MPASS(in_epoch(net_epoch_preempt) || mtx_owned(&(trunk)->lock)) #define TRUNK_WLOCK_ASSERT(trunk) mtx_assert(&(trunk)->lock, MA_OWNED); /* @@ -704,18 +703,17 @@ vlan_ifdetach(void *arg __unused, struct ifnet *ifp) static struct ifnet * vlan_trunkdev(struct ifnet *ifp) { - struct epoch_tracker et; struct ifvlan *ifv; + NET_EPOCH_ASSERT(); + if (ifp->if_type != IFT_L2VLAN) return (NULL); - NET_EPOCH_ENTER(et); ifv = ifp->if_softc; ifp = NULL; if (ifv->ifv_trunk) ifp = PARENT(ifv); - NET_EPOCH_EXIT(et); return (ifp); } @@ -787,21 +785,18 @@ vlan_setcookie(struct ifnet *ifp, void *cookie) static struct ifnet * vlan_devat(struct ifnet *ifp, uint16_t vid) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifvlan *ifv; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + trunk = ifp->if_vlantrunk; - if (trunk == NULL) { - NET_EPOCH_EXIT(et); + if (trunk == NULL) return (NULL); - } ifp = NULL; ifv = vlan_gethash(trunk, vid); if (ifv) ifp = ifv->ifv_ifp; - NET_EPOCH_EXIT(et); return (ifp); } @@ -1140,16 +1135,15 @@ vlan_init(void *foo __unused) static int vlan_transmit(struct ifnet *ifp, struct mbuf *m) { - struct epoch_tracker et; struct ifvlan *ifv; struct ifnet *p; int error, len, mcast; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + ifv = ifp->if_softc; if (TRUNK(ifv) == NULL) { if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); m_freem(m); return (ENETDOWN); } @@ -1169,7 +1163,6 @@ vlan_transmit(struct ifnet *ifp, struct mbuf *m) vst = mst_to_vst(mst); if (vst->tag->ifp != p) { if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); m_freem(m); return (EAGAIN); } @@ -1185,14 +1178,12 @@ vlan_transmit(struct ifnet *ifp, struct mbuf *m) */ if (!UP_AND_RUNNING(p)) { if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); m_freem(m); return (ENETDOWN); } if (!ether_8021q_frame(&m, ifp, p, ifv->ifv_vid, ifv->ifv_pcp)) { if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); return (0); } @@ -1206,7 +1197,6 @@ vlan_transmit(struct ifnet *ifp, struct mbuf *m) if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast); } else if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); return (error); } @@ -1214,19 +1204,17 @@ static int vlan_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, struct route *ro) { - struct epoch_tracker et; struct ifvlan *ifv; struct ifnet *p; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + ifv = ifp->if_softc; if (TRUNK(ifv) == NULL) { - NET_EPOCH_EXIT(et); m_freem(m); return (ENETDOWN); } p = PARENT(ifv); - NET_EPOCH_EXIT(et); return p->if_output(ifp, m, dst, ro); } @@ -1242,16 +1230,15 @@ vlan_qflush(struct ifnet *ifp __unused) static void vlan_input(struct ifnet *ifp, struct mbuf *m) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifvlan *ifv; struct m_tag *mtag; uint16_t vid, tag; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + trunk = ifp->if_vlantrunk; if (trunk == NULL) { - NET_EPOCH_EXIT(et); m_freem(m); return; } @@ -1274,7 +1261,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m) if (m->m_len < sizeof(*evl) && (m = m_pullup(m, sizeof(*evl))) == NULL) { if_printf(ifp, "cannot pullup VLAN header\n"); - NET_EPOCH_EXIT(et); return; } evl = mtod(m, struct ether_vlan_header *); @@ -1297,7 +1283,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m) __func__, ifp->if_xname, ifp->if_type); #endif if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); - NET_EPOCH_EXIT(et); m_freem(m); return; } @@ -1307,7 +1292,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m) ifv = vlan_gethash(trunk, vid); if (ifv == NULL || !UP_AND_RUNNING(ifv->ifv_ifp)) { - NET_EPOCH_EXIT(et); if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); m_freem(m); return; @@ -1327,7 +1311,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m) sizeof(uint8_t), M_NOWAIT); if (mtag == NULL) { if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); - NET_EPOCH_EXIT(et); m_freem(m); return; } @@ -1338,7 +1321,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m) m->m_pkthdr.rcvif = ifv->ifv_ifp; if_inc_counter(ifv->ifv_ifp, IFCOUNTER_IPACKETS, 1); - NET_EPOCH_EXIT(et); /* Pass it back through the parent's input routine. */ (*ifv->ifv_ifp->if_input)(ifv->ifv_ifp, m); @@ -1364,11 +1346,12 @@ vlan_lladdr_fn(void *arg, int pending __unused) static int vlan_config(struct ifvlan *ifv, struct ifnet *p, uint16_t vid) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifnet *ifp; int error = 0; + NET_EPOCH_ASSERT(); + /* * We can handle non-ethernet hardware types as long as * they handle the tagging and headers themselves. @@ -1469,9 +1452,7 @@ vlan_config(struct ifvlan *ifv, struct ifnet *p, uint1 ifp->if_link_state = p->if_link_state; - NET_EPOCH_ENTER(et); vlan_capabilities(ifv); - NET_EPOCH_EXIT(et); /* * Set up our interface address to reflect the underlying @@ -1643,17 +1624,14 @@ vlan_setflags(struct ifnet *ifp, int status) static void vlan_link_state(struct ifnet *ifp) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifvlan *ifv; - /* Called from a taskqueue_swi task, so we cannot sleep. */ - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + trunk = ifp->if_vlantrunk; - if (trunk == NULL) { - NET_EPOCH_EXIT(et); + if (trunk == NULL) return; - } TRUNK_WLOCK(trunk); VLAN_FOREACH(ifv, trunk) { @@ -1662,7 +1640,6 @@ vlan_link_state(struct ifnet *ifp) trunk->parent->if_link_state); } TRUNK_WUNLOCK(trunk); - NET_EPOCH_EXIT(et); } static void @@ -1674,8 +1651,9 @@ vlan_capabilities(struct ifvlan *ifv) int cap = 0, ena = 0, mena; u_long hwa = 0; - VLAN_SXLOCK_ASSERT(); NET_EPOCH_ASSERT(); + VLAN_SXLOCK_ASSERT(); + p = PARENT(ifv); ifp = ifv->ifv_ifp; @@ -1791,7 +1769,6 @@ vlan_capabilities(struct ifvlan *ifv) static void vlan_trunk_capabilities(struct ifnet *ifp) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifvlan *ifv; @@ -1801,11 +1778,8 @@ vlan_trunk_capabilities(struct ifnet *ifp) VLAN_SUNLOCK(); return; } - NET_EPOCH_ENTER(et); - VLAN_FOREACH(ifv, trunk) { + VLAN_FOREACH(ifv, trunk) vlan_capabilities(ifv); - } - NET_EPOCH_EXIT(et); VLAN_SUNLOCK(); } @@ -1820,6 +1794,8 @@ vlan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data struct vlanreq vlr; int error = 0; + NET_EPOCH_ASSERT(); + ifr = (struct ifreq *)data; ifa = (struct ifaddr *) data; ifv = ifp->if_softc; @@ -1996,13 +1972,8 @@ vlan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data VLAN_SLOCK(); ifv->ifv_capenable = ifr->ifr_reqcap; trunk = TRUNK(ifv); - if (trunk != NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); + if (trunk != NULL) vlan_capabilities(ifv); - NET_EPOCH_EXIT(et); - } VLAN_SUNLOCK(); break; Modified: head/sys/net/netisr.c ============================================================================== --- head/sys/net/netisr.c Mon Oct 7 20:41:55 2019 (r353291) +++ head/sys/net/netisr.c Mon Oct 7 22:40:05 2019 (r353292) @@ -861,6 +861,7 @@ static u_int netisr_process_workstream_proto(struct netisr_workstream *nwsp, u_int proto) { struct netisr_work local_npw, *npwp; + struct epoch_tracker et; u_int handled; struct mbuf *m; @@ -890,6 +891,7 @@ netisr_process_workstream_proto(struct netisr_workstre npwp->nw_len = 0; nwsp->nws_pendingbits &= ~(1 << proto); NWS_UNLOCK(nwsp); + NET_EPOCH_ENTER(et); while ((m = local_npw.nw_head) != NULL) { local_npw.nw_head = m->m_nextpkt; m->m_nextpkt = NULL; @@ -902,6 +904,7 @@ netisr_process_workstream_proto(struct netisr_workstre netisr_proto[proto].np_handler(m); CURVNET_RESTORE(); } + NET_EPOCH_EXIT(et); KASSERT(local_npw.nw_len == 0, ("%s(%u): len %u", __func__, proto, local_npw.nw_len)); if (netisr_proto[proto].np_drainedcpu) @@ -1088,6 +1091,7 @@ netisr_dispatch_src(u_int proto, uintptr_t source, str int dosignal, error; u_int cpuid, dispatch_policy; + NET_EPOCH_ASSERT(); KASSERT(proto < NETISR_MAXPROT, ("%s: invalid proto %u", __func__, proto)); #ifdef NETISR_LOCKING Modified: head/sys/net/route.c ============================================================================== --- head/sys/net/route.c Mon Oct 7 20:41:55 2019 (r353291) +++ head/sys/net/route.c Mon Oct 7 22:40:05 2019 (r353292) @@ -593,12 +593,12 @@ rtredirect_fib(struct sockaddr *dst, int error = 0; short *stat = NULL; struct rt_addrinfo info; - struct epoch_tracker et; struct ifaddr *ifa; struct rib_head *rnh; + NET_EPOCH_ASSERT(); + ifa = NULL; - NET_EPOCH_ENTER(et); rnh = rt_tables_get_rnh(fibnum, dst->sa_family); if (rnh == NULL) { error = EAFNOSUPPORT; @@ -693,7 +693,6 @@ done: if (rt) RTFREE_LOCKED(rt); out: - NET_EPOCH_EXIT(et); if (error) V_rtstat.rts_badredirect++; else if (stat != NULL) Modified: head/sys/net/rtsock.c ============================================================================== --- head/sys/net/rtsock.c Mon Oct 7 20:41:55 2019 (r353291) +++ head/sys/net/rtsock.c Mon Oct 7 22:40:05 2019 (r353292) @@ -560,6 +560,7 @@ route_output(struct mbuf *m, struct socket *so, ...) struct rib_head *rnh; struct rt_addrinfo info; struct sockaddr_storage ss; + struct epoch_tracker et; #ifdef INET6 struct sockaddr_in6 *sin6; int i, rti_need_deembed = 0; @@ -579,6 +580,7 @@ route_output(struct mbuf *m, struct socket *so, ...) return (ENOBUFS); if ((m->m_flags & M_PKTHDR) == 0) panic("route_output"); + NET_EPOCH_ENTER(et); len = m->m_pkthdr.len; if (len < sizeof(*rtm) || len != mtod(m, struct rt_msghdr *)->rtm_msglen) @@ -803,11 +805,11 @@ route_output(struct mbuf *m, struct socket *so, ...) NET_EPOCH_ENTER(et); ifa = ifa_ifwithnet(info.rti_info[RTAX_DST], 1, RT_ALL_FIBS); + NET_EPOCH_EXIT(et); if (ifa != NULL) rt_maskedcopy(ifa->ifa_addr, &laddr, ifa->ifa_netmask); - NET_EPOCH_EXIT(et); } else rt_maskedcopy(rt->rt_ifa->ifa_addr, &laddr, @@ -898,6 +900,7 @@ report: } flush: + NET_EPOCH_EXIT(et); if (rt != NULL) RTFREE(rt); /* @@ -1761,11 +1764,9 @@ sysctl_iflist(int af, struct walkarg *w) struct rt_addrinfo info; int len, error = 0; struct sockaddr_storage ss; - struct epoch_tracker et; bzero((caddr_t)&info, sizeof(info)); bzero(&ifd, sizeof(ifd)); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (w->w_arg && w->w_arg != ifp->if_index) continue; @@ -1815,7 +1816,6 @@ sysctl_iflist(int af, struct walkarg *w) info.rti_info[RTAX_BRD] = NULL; } done: - NET_EPOCH_EXIT(et); return (error); } @@ -1823,16 +1823,16 @@ static int sysctl_ifmalist(int af, struct walkarg *w) { struct rt_addrinfo info; - struct epoch_tracker et; struct ifaddr *ifa; struct ifmultiaddr *ifma; struct ifnet *ifp; int error, len; + NET_EPOCH_ASSERT(); + error = 0; bzero((caddr_t)&info, sizeof(info)); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (w->w_arg && w->w_arg != ifp->if_index) continue; @@ -1867,7 +1867,6 @@ sysctl_ifmalist(int af, struct walkarg *w) if (error != 0) break; } - NET_EPOCH_EXIT(et); return (error); } @@ -1875,6 +1874,7 @@ static int sysctl_rtsock(SYSCTL_HANDLER_ARGS) { RIB_RLOCK_TRACKER; + struct epoch_tracker et; int *name = (int *)arg1; u_int namelen = arg2; struct rib_head *rnh = NULL; /* silence compiler. */ @@ -1918,8 +1918,8 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS) w.w_tmemsize = 65536; w.w_tmem = malloc(w.w_tmemsize, M_TEMP, M_WAITOK); + NET_EPOCH_ENTER(et); switch (w.w_op) { - case NET_RT_DUMP: case NET_RT_FLAGS: if (af == 0) { /* dump all tables */ @@ -1946,13 +1946,9 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS) for (error = 0; error == 0 && i <= lim; i++) { rnh = rt_tables_get_rnh(fib, i); if (rnh != NULL) { - struct epoch_tracker et; - RIB_RLOCK(rnh); - NET_EPOCH_ENTER(et); error = rnh->rnh_walktree(&rnh->head, sysctl_dumpentry, &w); - NET_EPOCH_EXIT(et); RIB_RUNLOCK(rnh); } else if (af != 0) error = EAFNOSUPPORT; @@ -1968,6 +1964,7 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS) error = sysctl_ifmalist(af, &w); break; } + NET_EPOCH_EXIT(et); free(w.w_tmem, M_TEMP); return (error); *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201910072240.x97Me60x065650>