Date: Mon, 3 Nov 2014 12:38:42 +0000 (UTC) From: Hans Petter Selasky <hselasky@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-9@freebsd.org Subject: svn commit: r274044 - in stable/9/sys: dev/oce dev/xen/netfront kern net netinet ofed/drivers/net/mlx4 sys Message-ID: <201411031238.sA3Ccgc9077380@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: hselasky Date: Mon Nov 3 12:38:41 2014 New Revision: 274044 URL: https://svnweb.freebsd.org/changeset/base/274044 Log: MFC r271946 and r272595: Improve transmit sending offload, TSO, algorithm in general. This change allows all HCAs from Mellanox Technologies to function properly when TSO is enabled. See r271946 and r272595 for more details about this commit. Sponsored by: Mellanox Technologies Modified: stable/9/sys/dev/oce/oce_if.c stable/9/sys/dev/oce/oce_if.h stable/9/sys/dev/xen/netfront/netfront.c stable/9/sys/kern/uipc_sockbuf.c stable/9/sys/net/if.c stable/9/sys/net/if_lagg.c stable/9/sys/net/if_var.h stable/9/sys/net/if_vlan.c stable/9/sys/netinet/tcp_input.c stable/9/sys/netinet/tcp_output.c stable/9/sys/netinet/tcp_subr.c stable/9/sys/netinet/tcp_var.h stable/9/sys/ofed/drivers/net/mlx4/en_netdev.c stable/9/sys/sys/sockbuf.h Directory Properties: stable/9/sys/ (props changed) stable/9/sys/dev/ (props changed) stable/9/sys/net/ (props changed) stable/9/sys/sys/ (props changed) Modified: stable/9/sys/dev/oce/oce_if.c ============================================================================== --- stable/9/sys/dev/oce/oce_if.c Mon Nov 3 12:38:29 2014 (r274043) +++ stable/9/sys/dev/oce/oce_if.c Mon Nov 3 12:38:41 2014 (r274044) @@ -1734,7 +1734,9 @@ oce_attach_ifp(POCE_SOFTC sc) sc->ifp->if_baudrate = IF_Gbps(10UL); #if __FreeBSD_version >= 1000000 - sc->ifp->if_hw_tsomax = OCE_MAX_TSO_SIZE; + sc->ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); + sc->ifp->if_hw_tsomaxsegcount = OCE_MAX_TX_ELEMENTS; + sc->ifp->if_hw_tsomaxsegsize = 4096; #endif ether_ifattach(sc->ifp, sc->macaddr.mac_addr); Modified: stable/9/sys/dev/oce/oce_if.h ============================================================================== --- stable/9/sys/dev/oce/oce_if.h Mon Nov 3 12:38:29 2014 (r274043) +++ stable/9/sys/dev/oce/oce_if.h Mon Nov 3 12:38:41 2014 (r274044) @@ -150,7 +150,6 @@ extern int mp_ncpus; /* system's total #define OCE_MAX_TX_ELEMENTS 29 #define OCE_MAX_TX_DESC 1024 #define OCE_MAX_TX_SIZE 65535 -#define OCE_MAX_TSO_SIZE (65535 - ETHER_HDR_LEN) #define OCE_MAX_RX_SIZE 4096 #define OCE_MAX_RQ_POSTS 255 #define OCE_DEFAULT_PROMISCUOUS 0 Modified: stable/9/sys/dev/xen/netfront/netfront.c ============================================================================== --- stable/9/sys/dev/xen/netfront/netfront.c Mon Nov 3 12:38:29 2014 (r274043) +++ stable/9/sys/dev/xen/netfront/netfront.c Mon Nov 3 12:38:41 2014 (r274044) @@ -135,7 +135,6 @@ static const int MODPARM_rx_flip = 0; * to mirror the Linux MAX_SKB_FRAGS constant. */ #define MAX_TX_REQ_FRAGS (65536 / PAGE_SIZE + 2) -#define NF_TSO_MAXBURST ((IP_MAXPACKET / PAGE_SIZE) * MCLBYTES) #define RX_COPY_THRESHOLD 256 @@ -2103,7 +2102,9 @@ create_netdev(device_t dev) ifp->if_hwassist = XN_CSUM_FEATURES; ifp->if_capabilities = IFCAP_HWCSUM; - ifp->if_hw_tsomax = NF_TSO_MAXBURST; + ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); + ifp->if_hw_tsomaxsegcount = MAX_TX_REQ_FRAGS; + ifp->if_hw_tsomaxsegsize = PAGE_SIZE; ether_ifattach(ifp, np->mac); callout_init(&np->xn_stat_ch, CALLOUT_MPSAFE); Modified: stable/9/sys/kern/uipc_sockbuf.c ============================================================================== --- stable/9/sys/kern/uipc_sockbuf.c Mon Nov 3 12:38:29 2014 (r274043) +++ stable/9/sys/kern/uipc_sockbuf.c Mon Nov 3 12:38:41 2014 (r274044) @@ -989,6 +989,37 @@ sbsndptr(struct sockbuf *sb, u_int off, } /* + * Return the first mbuf and the mbuf data offset for the provided + * send offset without changing the "sb_sndptroff" field. + */ +struct mbuf * +sbsndmbuf(struct sockbuf *sb, u_int off, u_int *moff) +{ + struct mbuf *m; + + KASSERT(sb->sb_mb != NULL, ("%s: sb_mb is NULL", __func__)); + + /* + * If the "off" is below the stored offset, which happens on + * retransmits, just use "sb_mb": + */ + if (sb->sb_sndptr == NULL || sb->sb_sndptroff > off) { + m = sb->sb_mb; + } else { + m = sb->sb_sndptr; + off -= sb->sb_sndptroff; + } + while (off > 0 && m != NULL) { + if (off < m->m_len) + break; + off -= m->m_len; + m = m->m_next; + } + *moff = off; + return (m); +} + +/* * Drop a record off the front of a sockbuf and move the next record to the * front. */ Modified: stable/9/sys/net/if.c ============================================================================== --- stable/9/sys/net/if.c Mon Nov 3 12:38:29 2014 (r274043) +++ stable/9/sys/net/if.c Mon Nov 3 12:38:41 2014 (r274044) @@ -599,6 +599,57 @@ if_attach(struct ifnet *ifp) if_attach_internal(ifp, 0); } +/* + * Compute the least common TSO limit. + */ +void +if_hw_tsomax_common(struct ifnet *ifp, struct ifnet_hw_tsomax *pmax) +{ + /* + * 1) If there is no limit currently, take the limit from + * the network adapter. + * + * 2) If the network adapter has a limit below the current + * limit, apply it. + */ + if (pmax->tsomaxbytes == 0 || (ifp->if_hw_tsomax != 0 && + ifp->if_hw_tsomax < pmax->tsomaxbytes)) { + pmax->tsomaxbytes = ifp->if_hw_tsomax; + } + if (pmax->tsomaxsegcount == 0 || (ifp->if_hw_tsomaxsegcount != 0 && + ifp->if_hw_tsomaxsegcount < pmax->tsomaxsegcount)) { + pmax->tsomaxsegcount = ifp->if_hw_tsomaxsegcount; + } + if (pmax->tsomaxsegsize == 0 || (ifp->if_hw_tsomaxsegsize != 0 && + ifp->if_hw_tsomaxsegsize < pmax->tsomaxsegsize)) { + pmax->tsomaxsegsize = ifp->if_hw_tsomaxsegsize; + } +} + +/* + * Update TSO limit of a network adapter. + * + * Returns zero if no change. Else non-zero. + */ +int +if_hw_tsomax_update(struct ifnet *ifp, struct ifnet_hw_tsomax *pmax) +{ + int retval = 0; + if (ifp->if_hw_tsomax != pmax->tsomaxbytes) { + ifp->if_hw_tsomax = pmax->tsomaxbytes; + retval++; + } + if (ifp->if_hw_tsomaxsegsize != pmax->tsomaxsegsize) { + ifp->if_hw_tsomaxsegsize = pmax->tsomaxsegsize; + retval++; + } + if (ifp->if_hw_tsomaxsegcount != pmax->tsomaxsegcount) { + ifp->if_hw_tsomaxsegcount = pmax->tsomaxsegcount; + retval++; + } + return (retval); +} + static void if_attach_internal(struct ifnet *ifp, int vmove) { @@ -673,13 +724,36 @@ if_attach_internal(struct ifnet *ifp, in ifp->if_broadcastaddr = NULL; #if defined(INET) || defined(INET6) - /* Initialize to max value. */ - if (ifp->if_hw_tsomax == 0) - ifp->if_hw_tsomax = min(IP_MAXPACKET, 32 * MCLBYTES - + /* Use defaults for TSO, if nothing is set */ + if (ifp->if_hw_tsomax == 0 && + ifp->if_hw_tsomaxsegcount == 0 && + ifp->if_hw_tsomaxsegsize == 0) { + /* + * The TSO defaults needs to be such that an + * NFS mbuf list of 35 mbufs totalling just + * below 64K works and that a chain of mbufs + * can be defragged into at most 32 segments: + */ + ifp->if_hw_tsomax = min(IP_MAXPACKET, (32 * MCLBYTES) - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN)); - KASSERT(ifp->if_hw_tsomax <= IP_MAXPACKET && - ifp->if_hw_tsomax >= IP_MAXPACKET / 8, - ("%s: tsomax outside of range", __func__)); + ifp->if_hw_tsomaxsegcount = 35; + ifp->if_hw_tsomaxsegsize = 2048; /* 2K */ + + /* XXX some drivers set IFCAP_TSO after ethernet attach */ + if (ifp->if_capabilities & IFCAP_TSO) { + if_printf(ifp, "Using defaults for TSO: %u/%u/%u\n", + ifp->if_hw_tsomax, + ifp->if_hw_tsomaxsegcount, + ifp->if_hw_tsomaxsegsize); + } + } + /* + * If the "if_hw_tsomax" limit is set, check if it is + * too small: + */ + KASSERT(ifp->if_hw_tsomax == 0 || + ifp->if_hw_tsomax >= (IP_MAXPACKET / 8), + ("%s: if_hw_tsomax is outside of range", __func__)); #endif } #ifdef VIMAGE Modified: stable/9/sys/net/if_lagg.c ============================================================================== --- stable/9/sys/net/if_lagg.c Mon Nov 3 12:38:29 2014 (r274043) +++ stable/9/sys/net/if_lagg.c Mon Nov 3 12:38:41 2014 (r274044) @@ -406,23 +406,18 @@ lagg_capabilities(struct lagg_softc *sc) struct lagg_port *lp; int cap = ~0, ena = ~0; u_long hwa = ~0UL; -#if defined(INET) || defined(INET6) - u_int hw_tsomax = IP_MAXPACKET; /* Initialize to the maximum value. */ -#else - u_int hw_tsomax = ~0; /* if_hw_tsomax is only for INET/INET6, but.. */ -#endif + struct ifnet_hw_tsomax hw_tsomax; LAGG_WLOCK_ASSERT(sc); + memset(&hw_tsomax, 0, sizeof(hw_tsomax)); + /* Get capabilities from the lagg ports */ SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) { cap &= lp->lp_ifp->if_capabilities; ena &= lp->lp_ifp->if_capenable; hwa &= lp->lp_ifp->if_hwassist; - /* Set to the minimum value of the lagg ports. */ - if (lp->lp_ifp->if_hw_tsomax < hw_tsomax && - lp->lp_ifp->if_hw_tsomax > 0) - hw_tsomax = lp->lp_ifp->if_hw_tsomax; + if_hw_tsomax_common(lp->lp_ifp, &hw_tsomax); } cap = (cap == ~0 ? 0 : cap); ena = (ena == ~0 ? 0 : ena); @@ -431,11 +426,10 @@ lagg_capabilities(struct lagg_softc *sc) if (sc->sc_ifp->if_capabilities != cap || sc->sc_ifp->if_capenable != ena || sc->sc_ifp->if_hwassist != hwa || - sc->sc_ifp->if_hw_tsomax != hw_tsomax) { + if_hw_tsomax_update(sc->sc_ifp, &hw_tsomax) != 0) { sc->sc_ifp->if_capabilities = cap; sc->sc_ifp->if_capenable = ena; sc->sc_ifp->if_hwassist = hwa; - sc->sc_ifp->if_hw_tsomax = hw_tsomax; getmicrotime(&sc->sc_ifp->if_lastchange); if (sc->sc_ifflags & IFF_DEBUG) Modified: stable/9/sys/net/if_var.h ============================================================================== --- stable/9/sys/net/if_var.h Mon Nov 3 12:38:29 2014 (r274043) +++ stable/9/sys/net/if_var.h Mon Nov 3 12:38:41 2014 (r274044) @@ -111,6 +111,12 @@ struct ifqueue { struct mtx ifq_mtx; }; +struct ifnet_hw_tsomax { + u_int tsomaxbytes; /* TSO total burst length limit in bytes */ + u_int tsomaxsegcount; /* TSO maximum segment count */ + u_int tsomaxsegsize; /* TSO maximum segment size in bytes */ +}; + /* * Structure defining a network interface. * @@ -205,11 +211,18 @@ struct ifnet { * be used with care where binary compatibility is required. */ char if_cspare[3]; - u_int if_hw_tsomax; /* tso burst length limit, the minmum - * is (IP_MAXPACKET / 8). - * XXXAO: Have to find a better place - * for it eventually. */ - int if_ispare[3]; + u_int if_hw_tsomax; /* TSO total burst length + * limit in bytes. A value of + * zero means no limit. Have + * to find a better place for + * it eventually. */ + int if_ispare[1]; + /* + * TSO fields for segment limits. If a field is zero below, + * there is no limit: + */ + u_int if_hw_tsomaxsegcount; /* TSO maximum segment count */ + u_int if_hw_tsomaxsegsize; /* TSO maximum segment size in bytes */ void *if_pspare[8]; /* 1 netmap, 7 TDB */ }; @@ -966,6 +979,10 @@ int ether_poll_register(poll_handler_ int ether_poll_deregister(struct ifnet *ifp); #endif /* DEVICE_POLLING */ +/* TSO */ +void if_hw_tsomax_common(struct ifnet *, struct ifnet_hw_tsomax *); +int if_hw_tsomax_update(struct ifnet *, struct ifnet_hw_tsomax *); + #endif /* _KERNEL */ #endif /* !_NET_IF_VAR_H_ */ Modified: stable/9/sys/net/if_vlan.c ============================================================================== --- stable/9/sys/net/if_vlan.c Mon Nov 3 12:38:29 2014 (r274043) +++ stable/9/sys/net/if_vlan.c Mon Nov 3 12:38:41 2014 (r274044) @@ -1475,6 +1475,7 @@ vlan_capabilities(struct ifvlan *ifv) { struct ifnet *p = PARENT(ifv); struct ifnet *ifp = ifv->ifv_ifp; + struct ifnet_hw_tsomax hw_tsomax; TRUNK_LOCK_ASSERT(TRUNK(ifv)); @@ -1501,8 +1502,9 @@ vlan_capabilities(struct ifvlan *ifv) * propagate the hardware-assisted flag. TSO on VLANs * does not necessarily require hardware VLAN tagging. */ - if (p->if_hw_tsomax > 0) - ifp->if_hw_tsomax = p->if_hw_tsomax; + memset(&hw_tsomax, 0, sizeof(hw_tsomax)); + if_hw_tsomax_common(p, &hw_tsomax); + if_hw_tsomax_update(ifp, &hw_tsomax); if (p->if_capabilities & IFCAP_VLAN_HWTSO) ifp->if_capabilities |= p->if_capabilities & IFCAP_TSO; if (p->if_capenable & IFCAP_VLAN_HWTSO) { Modified: stable/9/sys/netinet/tcp_input.c ============================================================================== --- stable/9/sys/netinet/tcp_input.c Mon Nov 3 12:38:29 2014 (r274043) +++ stable/9/sys/netinet/tcp_input.c Mon Nov 3 12:38:41 2014 (r274044) @@ -3628,6 +3628,8 @@ tcp_mss(struct tcpcb *tp, int offer) if (cap.ifcap & CSUM_TSO) { tp->t_flags |= TF_TSO; tp->t_tsomax = cap.tsomax; + tp->t_tsomaxsegcount = cap.tsomaxsegcount; + tp->t_tsomaxsegsize = cap.tsomaxsegsize; } } Modified: stable/9/sys/netinet/tcp_output.c ============================================================================== --- stable/9/sys/netinet/tcp_output.c Mon Nov 3 12:38:29 2014 (r274043) +++ stable/9/sys/netinet/tcp_output.c Mon Nov 3 12:38:41 2014 (r274044) @@ -765,28 +765,112 @@ send: flags &= ~TH_FIN; if (tso) { + u_int if_hw_tsomax; + u_int if_hw_tsomaxsegcount; + u_int if_hw_tsomaxsegsize; + struct mbuf *mb; + u_int moff; + int max_len; + + /* extract TSO information */ + if_hw_tsomax = tp->t_tsomax; + if_hw_tsomaxsegcount = tp->t_tsomaxsegcount; + if_hw_tsomaxsegsize = tp->t_tsomaxsegsize; + + /* + * Limit a TSO burst to prevent it from + * overflowing or exceeding the maximum length + * allowed by the network interface: + */ KASSERT(ipoptlen == 0, ("%s: TSO can't do IP options", __func__)); /* - * Limit a burst to t_tsomax minus IP, - * TCP and options length to keep ip->ip_len - * from overflowing or exceeding the maximum - * length allowed by the network interface. + * Check if we should limit by maximum payload + * length: */ - if (len > tp->t_tsomax - hdrlen) { - len = tp->t_tsomax - hdrlen; - sendalot = 1; + if (if_hw_tsomax != 0) { + /* compute maximum TSO length */ + max_len = (if_hw_tsomax - hdrlen); + if (max_len <= 0) { + len = 0; + } else if (len > (u_int)max_len) { + sendalot = 1; + len = (u_int)max_len; + } + } + + /* + * Check if we should limit by maximum segment + * size and count: + */ + if (if_hw_tsomaxsegcount != 0 && + if_hw_tsomaxsegsize != 0) { + max_len = 0; + mb = sbsndmbuf(&so->so_snd, off, &moff); + + while (mb != NULL && (u_int)max_len < len) { + u_int mlen; + u_int frags; + + /* + * Get length of mbuf fragment + * and how many hardware frags, + * rounded up, it would use: + */ + mlen = (mb->m_len - moff); + frags = howmany(mlen, + if_hw_tsomaxsegsize); + + /* Handle special case: Zero Length Mbuf */ + if (frags == 0) + frags = 1; + + /* + * Check if the fragment limit + * will be reached or exceeded: + */ + if (frags >= if_hw_tsomaxsegcount) { + max_len += min(mlen, + if_hw_tsomaxsegcount * + if_hw_tsomaxsegsize); + break; + } + max_len += mlen; + if_hw_tsomaxsegcount -= frags; + moff = 0; + mb = mb->m_next; + } + if (max_len <= 0) { + len = 0; + } else if (len > (u_int)max_len) { + sendalot = 1; + len = (u_int)max_len; + } } /* * Prevent the last segment from being - * fractional unless the send sockbuf can - * be emptied. + * fractional unless the send sockbuf can be + * emptied: + */ + max_len = (tp->t_maxopd - optlen); + if ((off + len) < so->so_snd.sb_cc) { + moff = len % (u_int)max_len; + if (moff != 0) { + len -= moff; + sendalot = 1; + } + } + + /* + * In case there are too many small fragments + * don't use TSO: */ - if (sendalot && off + len < so->so_snd.sb_cc) { - len -= len % (tp->t_maxopd - optlen); + if (len <= (u_int)max_len) { + len = (u_int)max_len; sendalot = 1; + tso = 0; } /* Modified: stable/9/sys/netinet/tcp_subr.c ============================================================================== --- stable/9/sys/netinet/tcp_subr.c Mon Nov 3 12:38:29 2014 (r274043) +++ stable/9/sys/netinet/tcp_subr.c Mon Nov 3 12:38:41 2014 (r274044) @@ -1744,6 +1744,8 @@ tcp_maxmtu(struct in_conninfo *inc, stru ifp->if_hwassist & CSUM_TSO) { cap->ifcap |= CSUM_TSO; cap->tsomax = ifp->if_hw_tsomax; + cap->tsomaxsegcount = ifp->if_hw_tsomaxsegcount; + cap->tsomaxsegsize = ifp->if_hw_tsomaxsegsize; } } RTFREE(sro.ro_rt); @@ -1783,6 +1785,8 @@ tcp_maxmtu6(struct in_conninfo *inc, str ifp->if_hwassist & CSUM_TSO) { cap->ifcap |= CSUM_TSO; cap->tsomax = ifp->if_hw_tsomax; + cap->tsomaxsegcount = ifp->if_hw_tsomaxsegcount; + cap->tsomaxsegsize = ifp->if_hw_tsomaxsegsize; } } RTFREE(sro6.ro_rt); Modified: stable/9/sys/netinet/tcp_var.h ============================================================================== --- stable/9/sys/netinet/tcp_var.h Mon Nov 3 12:38:29 2014 (r274043) +++ stable/9/sys/netinet/tcp_var.h Mon Nov 3 12:38:41 2014 (r274044) @@ -212,7 +212,9 @@ struct tcpcb { uint32_t t_ispare[7]; /* 5 UTO, 2 TBD */ void *t_pspare2[4]; /* 4 TBD */ - uint64_t _pad[6]; /* 6 TBD (1-2 CC/RTT?) */ + uint64_t _pad[5]; /* 5 TBD (1-2 CC/RTT?) */ + uint32_t t_tsomaxsegcount; /* TSO maximum segment count */ + uint32_t t_tsomaxsegsize; /* TSO maximum segment size in bytes */ }; /* @@ -333,6 +335,8 @@ struct hc_metrics_lite { /* must stay in struct tcp_ifcap { int ifcap; u_int tsomax; + u_int tsomaxsegcount; + u_int tsomaxsegsize; }; #ifndef _NETINET_IN_PCB_H_ Modified: stable/9/sys/ofed/drivers/net/mlx4/en_netdev.c ============================================================================== --- stable/9/sys/ofed/drivers/net/mlx4/en_netdev.c Mon Nov 3 12:38:29 2014 (r274043) +++ stable/9/sys/ofed/drivers/net/mlx4/en_netdev.c Mon Nov 3 12:38:41 2014 (r274044) @@ -2106,12 +2106,12 @@ int mlx4_en_init_netdev(struct mlx4_en_d if (mdev->LSO_support) dev->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6 | IFCAP_VLAN_HWTSO; -#if 0 + /* set TSO limits so that we don't have to drop TX packets */ dev->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); dev->if_hw_tsomaxsegcount = 16; dev->if_hw_tsomaxsegsize = 65536; /* XXX can do up to 4GByte */ -#endif + dev->if_capenable = dev->if_capabilities; dev->if_hwassist = 0; Modified: stable/9/sys/sys/sockbuf.h ============================================================================== --- stable/9/sys/sys/sockbuf.h Mon Nov 3 12:38:29 2014 (r274043) +++ stable/9/sys/sys/sockbuf.h Mon Nov 3 12:38:41 2014 (r274044) @@ -155,6 +155,8 @@ int sbreserve_locked(struct sockbuf *sb, struct thread *td); struct mbuf * sbsndptr(struct sockbuf *sb, u_int off, u_int len, u_int *moff); +struct mbuf * + sbsndmbuf(struct sockbuf *sb, u_int off, u_int *moff); void sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb); int sbwait(struct sockbuf *sb); int sblock(struct sockbuf *sb, int flags);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201411031238.sA3Ccgc9077380>