Date: Wed, 23 Dec 2020 20:30:09 GMT From: Oleksandr Tymoshenko <gonzo@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: e52326210786 - [if_dwc] add support for multi-descriptor packets in TX path Message-ID: <202012232030.0BNKU9nF047368@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by gonzo: URL: https://cgit.FreeBSD.org/src/commit/?id=e523262107865130e40fb19f7c3c571c8dd0b252 commit e523262107865130e40fb19f7c3c571c8dd0b252 Author: Oleksandr Tymoshenko <gonzo@bluezbox.com> AuthorDate: 2020-12-23 19:43:46 +0000 Commit: Oleksandr Tymoshenko <gonzo@FreeBSD.org> CommitDate: 2020-12-23 20:29:29 +0000 [if_dwc] add support for multi-descriptor packets in TX path Original if_dwc driver used m_defrag as an implementation shortcut but on 1000Mb networks it affects performance. Implement multi-descriptor support for TX path. Tested on RK3399-Firefly, patch adds ~15% of network throughput. Reviewed By: manu Differential Revision: https://reviews.freebsd.org/D27520 --- sys/dev/dwc/if_dwc.c | 146 +++++++++++++++++++++++++++++++++++------------- sys/dev/dwc/if_dwcvar.h | 13 ++++- 2 files changed, 116 insertions(+), 43 deletions(-) diff --git a/sys/dev/dwc/if_dwc.c b/sys/dev/dwc/if_dwc.c index ee871c268ea6..776d0d0dc392 100644 --- a/sys/dev/dwc/if_dwc.c +++ b/sys/dev/dwc/if_dwc.c @@ -627,7 +627,7 @@ dwc_get1paddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error) inline static void dwc_setup_txdesc(struct dwc_softc *sc, int idx, bus_addr_t paddr, - uint32_t len, uint32_t flags) + uint32_t len, uint32_t flags, bool first, bool last) { uint32_t desc0, desc1; @@ -635,55 +635,72 @@ dwc_setup_txdesc(struct dwc_softc *sc, int idx, bus_addr_t paddr, if (paddr == 0 || len == 0) { desc0 = 0; desc1 = 0; - --sc->txcount; + --sc->tx_desccount; } else { if (sc->mactype != DWC_GMAC_EXT_DESC) { desc0 = 0; - desc1 = NTDESC1_TCH | NTDESC1_FS | NTDESC1_LS | - NTDESC1_IC | len | flags; + desc1 = NTDESC1_TCH | len | flags; + if (first) + desc1 |= NTDESC1_FS; + if (last) + desc1 |= NTDESC1_LS | NTDESC1_IC; } else { - desc0 = ETDESC0_TCH | ETDESC0_FS | ETDESC0_LS | - ETDESC0_IC | flags; + desc0 = ETDESC0_TCH | flags; + if (first) + desc0 |= ETDESC0_FS; + if (last) + desc0 |= ETDESC0_LS | ETDESC0_IC; desc1 = len; } - ++sc->txcount; + ++sc->tx_desccount; } sc->txdesc_ring[idx].addr1 = (uint32_t)(paddr); sc->txdesc_ring[idx].desc0 = desc0; sc->txdesc_ring[idx].desc1 = desc1; +} - if (paddr && len) { - wmb(); - sc->txdesc_ring[idx].desc0 |= TDESC0_OWN; - wmb(); - } +inline static void +dwc_set_owner(struct dwc_softc *sc, int idx) +{ + wmb(); + sc->txdesc_ring[idx].desc0 |= TDESC0_OWN; + wmb(); } static int dwc_setup_txbuf(struct dwc_softc *sc, int idx, struct mbuf **mp) { - struct bus_dma_segment seg; + struct bus_dma_segment segs[TX_MAP_MAX_SEGS]; int error, nsegs; struct mbuf * m; uint32_t flags = 0; + int i; + int first, last; - if ((m = m_defrag(*mp, M_NOWAIT)) == NULL) + error = bus_dmamap_load_mbuf_sg(sc->txbuf_tag, sc->txbuf_map[idx].map, + *mp, segs, &nsegs, 0); + if (error == EFBIG) { + /* + * The map may be partially mapped from the first call. + * Make sure to reset it. + */ + bus_dmamap_unload(sc->txbuf_tag, sc->txbuf_map[idx].map); + if ((m = m_defrag(*mp, M_NOWAIT)) == NULL) + return (ENOMEM); + *mp = m; + error = bus_dmamap_load_mbuf_sg(sc->txbuf_tag, sc->txbuf_map[idx].map, + *mp, segs, &nsegs, 0); + } + if (error != 0) return (ENOMEM); - *mp = m; - error = bus_dmamap_load_mbuf_sg(sc->txbuf_tag, sc->txbuf_map[idx].map, - m, &seg, &nsegs, 0); - if (error != 0) { + if (sc->tx_desccount + nsegs > TX_DESC_COUNT) { + bus_dmamap_unload(sc->txbuf_tag, sc->txbuf_map[idx].map); return (ENOMEM); } - KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs)); - - bus_dmamap_sync(sc->txbuf_tag, sc->txbuf_map[idx].map, - BUS_DMASYNC_PREWRITE); - - sc->txbuf_map[idx].mbuf = m; + m = *mp; if ((m->m_pkthdr.csum_flags & CSUM_IP) != 0) { if ((m->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_UDP)) != 0) { @@ -699,7 +716,27 @@ dwc_setup_txbuf(struct dwc_softc *sc, int idx, struct mbuf **mp) } } - dwc_setup_txdesc(sc, idx, seg.ds_addr, seg.ds_len, flags); + bus_dmamap_sync(sc->txbuf_tag, sc->txbuf_map[idx].map, + BUS_DMASYNC_PREWRITE); + + sc->txbuf_map[idx].mbuf = m; + + first = sc->tx_desc_head; + for (i = 0; i < nsegs; i++) { + dwc_setup_txdesc(sc, sc->tx_desc_head, + segs[i].ds_addr, segs[i].ds_len, + (i == 0) ? flags : 0, /* only first desc needs flags */ + (i == 0), + (i == nsegs - 1)); + if (i > 0) + dwc_set_owner(sc, sc->tx_desc_head); + last = sc->tx_desc_head; + sc->tx_desc_head = next_txidx(sc, sc->tx_desc_head); + } + + sc->txbuf_map[idx].last_desc_idx = last; + + dwc_set_owner(sc, first); return (0); } @@ -900,7 +937,8 @@ setup_dma(struct dwc_softc *sc) BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ - MCLBYTES, 1, /* maxsize, nsegments */ + MCLBYTES*TX_MAP_MAX_SEGS, /* maxsize */ + TX_MAP_MAX_SEGS, /* nsegments */ MCLBYTES, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockarg */ @@ -911,7 +949,7 @@ setup_dma(struct dwc_softc *sc) goto out; } - for (idx = 0; idx < TX_DESC_COUNT; idx++) { + for (idx = 0; idx < TX_MAP_COUNT; idx++) { error = bus_dmamap_create(sc->txbuf_tag, BUS_DMA_COHERENT, &sc->txbuf_map[idx].map); if (error != 0) { @@ -919,9 +957,11 @@ setup_dma(struct dwc_softc *sc) "could not create TX buffer DMA map.\n"); goto out; } - dwc_setup_txdesc(sc, idx, 0, 0, 0); } + for (idx = 0; idx < TX_DESC_COUNT; idx++) + dwc_setup_txdesc(sc, idx, 0, 0, 0, false, false); + /* * Set up RX descriptor ring, descriptors, dma maps, and mbufs. */ @@ -1029,7 +1069,12 @@ dwc_txstart_locked(struct dwc_softc *sc) enqueued = 0; for (;;) { - if (sc->txcount == (TX_DESC_COUNT - 1)) { + if (sc->tx_desccount > (TX_DESC_COUNT - TX_MAP_MAX_SEGS + 1)) { + if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); + break; + } + + if (sc->tx_mapcount == (TX_MAP_COUNT - 1)) { if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); break; } @@ -1037,12 +1082,14 @@ dwc_txstart_locked(struct dwc_softc *sc) m = if_dequeue(ifp); if (m == NULL) break; - if (dwc_setup_txbuf(sc, sc->tx_idx_head, &m) != 0) { + if (dwc_setup_txbuf(sc, sc->tx_map_head, &m) != 0) { if_sendq_prepend(ifp, m); + if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); break; } if_bpfmtap(ifp, m); - sc->tx_idx_head = next_txidx(sc, sc->tx_idx_head); + sc->tx_map_head = next_txidx(sc, sc->tx_map_head); + sc->tx_mapcount++; ++enqueued; } @@ -1193,28 +1240,46 @@ dwc_txfinish_locked(struct dwc_softc *sc) struct dwc_bufmap *bmap; struct dwc_hwdesc *desc; struct ifnet *ifp; + int idx, last_idx; + bool map_finished; DWC_ASSERT_LOCKED(sc); ifp = sc->ifp; - while (sc->tx_idx_tail != sc->tx_idx_head) { - desc = &sc->txdesc_ring[sc->tx_idx_tail]; - if ((desc->desc0 & TDESC0_OWN) != 0) + /* check if all descriptors of the map are done */ + while (sc->tx_map_tail != sc->tx_map_head) { + map_finished = true; + bmap = &sc->txbuf_map[sc->tx_map_tail]; + idx = sc->tx_desc_tail; + last_idx = next_txidx(sc, bmap->last_desc_idx); + while (idx != last_idx) { + desc = &sc->txdesc_ring[idx]; + if ((desc->desc0 & TDESC0_OWN) != 0) { + map_finished = false; + break; + } + idx = next_txidx(sc, idx); + } + + if (!map_finished) break; - bmap = &sc->txbuf_map[sc->tx_idx_tail]; bus_dmamap_sync(sc->txbuf_tag, bmap->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->txbuf_tag, bmap->map); m_freem(bmap->mbuf); bmap->mbuf = NULL; - dwc_setup_txdesc(sc, sc->tx_idx_tail, 0, 0, 0); - sc->tx_idx_tail = next_txidx(sc, sc->tx_idx_tail); + sc->tx_mapcount--; + while (sc->tx_desc_tail != last_idx) { + dwc_setup_txdesc(sc, sc->tx_desc_tail, 0, 0, 0, false, false); + sc->tx_desc_tail = next_txidx(sc, sc->tx_desc_tail); + } + sc->tx_map_tail = next_txidx(sc, sc->tx_map_tail); if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); } /* If there are no buffers outstanding, muzzle the watchdog. */ - if (sc->tx_idx_tail == sc->tx_idx_head) { + if (sc->tx_desc_tail == sc->tx_desc_head) { sc->tx_watchdog_count = 0; } } @@ -1503,7 +1568,8 @@ dwc_attach(device_t dev) sc = device_get_softc(dev); sc->dev = dev; sc->rx_idx = 0; - sc->txcount = TX_DESC_COUNT; + sc->tx_desccount = TX_DESC_COUNT; + sc->tx_mapcount = 0; sc->mii_clk = IF_DWC_MII_CLK(dev); sc->mactype = IF_DWC_MAC_TYPE(dev); @@ -1610,7 +1676,7 @@ dwc_attach(device_t dev) if_setstartfn(ifp, dwc_txstart); if_setioctlfn(ifp, dwc_ioctl); if_setinitfn(ifp, dwc_init); - if_setsendqlen(ifp, TX_DESC_COUNT - 1); + if_setsendqlen(ifp, TX_MAP_COUNT - 1); if_setsendqready(sc->ifp); if_sethwassist(sc->ifp, CSUM_IP | CSUM_UDP | CSUM_TCP); if_setcapabilities(sc->ifp, IFCAP_VLAN_MTU | IFCAP_HWCSUM); diff --git a/sys/dev/dwc/if_dwcvar.h b/sys/dev/dwc/if_dwcvar.h index 0470b29cb0e1..97ae0ea681c8 100644 --- a/sys/dev/dwc/if_dwcvar.h +++ b/sys/dev/dwc/if_dwcvar.h @@ -47,11 +47,15 @@ #define RX_DESC_COUNT 1024 #define RX_DESC_SIZE (sizeof(struct dwc_hwdesc) * RX_DESC_COUNT) #define TX_DESC_COUNT 1024 +#define TX_MAP_COUNT TX_DESC_COUNT #define TX_DESC_SIZE (sizeof(struct dwc_hwdesc) * TX_DESC_COUNT) +#define TX_MAP_MAX_SEGS 32 struct dwc_bufmap { bus_dmamap_t map; struct mbuf *mbuf; + /* Only used for TX descirptors */ + int last_desc_idx; }; struct dwc_softc { @@ -89,9 +93,12 @@ struct dwc_softc { bus_addr_t txdesc_ring_paddr; bus_dma_tag_t txbuf_tag; struct dwc_bufmap txbuf_map[TX_DESC_COUNT]; - uint32_t tx_idx_head; - uint32_t tx_idx_tail; - int txcount; + uint32_t tx_desc_head; + uint32_t tx_desc_tail; + uint32_t tx_map_head; + uint32_t tx_map_tail; + int tx_desccount; + int tx_mapcount; }; #endif /* __IF_DWCVAR_H__ */
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202012232030.0BNKU9nF047368>