Date: Wed, 12 Dec 2018 11:52:48 +0000 (UTC) From: Hans Petter Selasky <hselasky@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org Subject: svn commit: r341911 - stable/12/sys/dev/mlx4/mlx4_en Message-ID: <201812121152.wBCBqmJu063379@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: hselasky Date: Wed Dec 12 11:52:48 2018 New Revision: 341911 URL: https://svnweb.freebsd.org/changeset/base/341911 Log: MFC r341549: mlx4en: Add support for receiving all data using one or more MCLBYTES sized mbufs. Also when the MTU is greater than MCLBYTES. Sponsored by: Mellanox Technologies Modified: stable/12/sys/dev/mlx4/mlx4_en/en.h stable/12/sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c stable/12/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c Directory Properties: stable/12/ (props changed) Modified: stable/12/sys/dev/mlx4/mlx4_en/en.h ============================================================================== --- stable/12/sys/dev/mlx4/mlx4_en/en.h Wed Dec 12 11:51:06 2018 (r341910) +++ stable/12/sys/dev/mlx4/mlx4_en/en.h Wed Dec 12 11:52:48 2018 (r341911) @@ -75,6 +75,15 @@ #define MAX_RX_RINGS 128 #define MIN_RX_RINGS 4 #define TXBB_SIZE 64 + +#ifndef MLX4_EN_MAX_RX_SEGS +#define MLX4_EN_MAX_RX_SEGS 1 /* or 8 */ +#endif + +#ifndef MLX4_EN_MAX_RX_BYTES +#define MLX4_EN_MAX_RX_BYTES MCLBYTES +#endif + #define HEADROOM (2048 / TXBB_SIZE + 1) #define INIT_OWNER_BIT 0xffffffff #define STAMP_STRIDE 64 @@ -297,10 +306,12 @@ struct mlx4_en_tx_ring { }; struct mlx4_en_rx_desc { - /* actual number of entries depends on rx ring stride */ - struct mlx4_wqe_data_seg data[0]; + struct mlx4_wqe_data_seg data[MLX4_EN_MAX_RX_SEGS]; }; +/* the size of the structure above must be power of two */ +CTASSERT(powerof2(sizeof(struct mlx4_en_rx_desc))); + struct mlx4_en_rx_mbuf { bus_dmamap_t dma_map; struct mbuf *mbuf; @@ -309,7 +320,7 @@ struct mlx4_en_rx_mbuf { struct mlx4_en_rx_spare { bus_dmamap_t dma_map; struct mbuf *mbuf; - u64 paddr_be; + bus_dma_segment_t segs[MLX4_EN_MAX_RX_SEGS]; }; struct mlx4_en_rx_ring { @@ -319,7 +330,6 @@ struct mlx4_en_rx_ring { u32 size ; /* number of Rx descs*/ u32 actual_size; u32 size_mask; - u16 stride; u16 log_stride; u16 cqn; /* index of port CQ associated with this ring */ u32 prod; @@ -327,6 +337,7 @@ struct mlx4_en_rx_ring { u32 buf_size; u8 fcs_del; u32 rx_mb_size; + u32 rx_mr_key_be; int qpn; u8 *buf; struct mlx4_en_rx_mbuf *mbuf; @@ -559,7 +570,6 @@ struct mlx4_en_priv { int registered; int gone; int allocated; - int stride; unsigned char current_mac[ETH_ALEN + 2]; u64 mac; int mac_index; @@ -805,7 +815,7 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, u32 size, int node); void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring **pring, - u32 size, u16 stride); + u32 size); void mlx4_en_rx_que(void *context, int pending); int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv); void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, Modified: stable/12/sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c ============================================================================== --- stable/12/sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c Wed Dec 12 11:51:06 2018 (r341910) +++ stable/12/sys/dev/mlx4/mlx4_en/mlx4_en_netdev.c Wed Dec 12 11:52:48 2018 (r341911) @@ -1683,7 +1683,7 @@ void mlx4_en_free_resources(struct mlx4_en_priv *priv) for (i = 0; i < priv->rx_ring_num; i++) { if (priv->rx_ring[i]) mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i], - priv->prof->rx_ring_size, priv->stride); + priv->prof->rx_ring_size); if (priv->rx_cq[i]) mlx4_en_destroy_cq(priv, &priv->rx_cq[i]); } @@ -1734,8 +1734,7 @@ err: for (i = 0; i < priv->rx_ring_num; i++) { if (priv->rx_ring[i]) mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i], - prof->rx_ring_size, - priv->stride); + prof->rx_ring_size); if (priv->rx_cq[i]) mlx4_en_destroy_cq(priv, &priv->rx_cq[i]); } @@ -2236,9 +2235,6 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int err = -EINVAL; goto out; } - - priv->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + - DS_SIZE); mlx4_en_sysctl_conf(priv); Modified: stable/12/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c ============================================================================== --- stable/12/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c Wed Dec 12 11:51:06 2018 (r341910) +++ stable/12/sys/dev/mlx4/mlx4_en/mlx4_en_rx.c Wed Dec 12 11:52:48 2018 (r341911) @@ -44,14 +44,13 @@ #include "en.h" - +#if (MLX4_EN_MAX_RX_SEGS == 1) static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, int index) { - struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *) - (ring->buf + (ring->stride * index)); - int possible_frags; + struct mlx4_en_rx_desc *rx_desc = + ((struct mlx4_en_rx_desc *)ring->buf) + index; int i; /* Set size and memtype fields */ @@ -63,38 +62,75 @@ static void mlx4_en_init_rx_desc(struct mlx4_en_priv * * stride, remaining (unused) fragments must be padded with * null address/size and a special memory key: */ - possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE; - for (i = 1; i < possible_frags; i++) { + for (i = 1; i < MLX4_EN_MAX_RX_SEGS; i++) { rx_desc->data[i].byte_count = 0; rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); rx_desc->data[i].addr = 0; } } +#endif +static inline struct mbuf * +mlx4_en_alloc_mbuf(struct mlx4_en_rx_ring *ring) +{ + struct mbuf *mb; + +#if (MLX4_EN_MAX_RX_SEGS == 1) + mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); + if (likely(mb != NULL)) + mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size; +#else + mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MLX4_EN_MAX_RX_BYTES); + if (likely(mb != NULL)) { + struct mbuf *mb_head = mb; + int i; + + mb->m_len = MLX4_EN_MAX_RX_BYTES; + mb->m_pkthdr.len = MLX4_EN_MAX_RX_BYTES; + + for (i = 1; i != MLX4_EN_MAX_RX_SEGS; i++) { + if (mb_head->m_pkthdr.len >= ring->rx_mb_size) + break; + mb = (mb->m_next = m_getjcl(M_NOWAIT, MT_DATA, 0, MLX4_EN_MAX_RX_BYTES)); + if (unlikely(mb == NULL)) { + m_freem(mb_head); + return (NULL); + } + mb->m_len = MLX4_EN_MAX_RX_BYTES; + mb_head->m_pkthdr.len += MLX4_EN_MAX_RX_BYTES; + } + /* rewind to first mbuf in chain */ + mb = mb_head; + } +#endif + return (mb); +} + static int -mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, - __be64 *pdma, struct mlx4_en_rx_mbuf *mb_list) +mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_desc *rx_desc, + struct mlx4_en_rx_mbuf *mb_list) { - bus_dma_segment_t segs[1]; + bus_dma_segment_t segs[MLX4_EN_MAX_RX_SEGS]; bus_dmamap_t map; struct mbuf *mb; int nsegs; int err; +#if (MLX4_EN_MAX_RX_SEGS != 1) + int i; +#endif /* try to allocate a new spare mbuf */ if (unlikely(ring->spare.mbuf == NULL)) { - mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); + mb = mlx4_en_alloc_mbuf(ring); if (unlikely(mb == NULL)) return (-ENOMEM); - /* setup correct length */ - mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size; /* make sure IP header gets aligned */ m_adj(mb, MLX4_NET_IP_ALIGN); /* load spare mbuf into BUSDMA */ err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, ring->spare.dma_map, - mb, segs, &nsegs, BUS_DMA_NOWAIT); + mb, ring->spare.segs, &nsegs, BUS_DMA_NOWAIT); if (unlikely(err != 0)) { m_freem(mb); return (err); @@ -102,8 +138,14 @@ mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, /* store spare info */ ring->spare.mbuf = mb; - ring->spare.paddr_be = cpu_to_be64(segs[0].ds_addr); +#if (MLX4_EN_MAX_RX_SEGS != 1) + /* zero remaining segs */ + for (i = nsegs; i != MLX4_EN_MAX_RX_SEGS; i++) { + ring->spare.segs[i].ds_addr = 0; + ring->spare.segs[i].ds_len = 0; + } +#endif bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, BUS_DMASYNC_PREREAD); } @@ -115,13 +157,10 @@ mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, bus_dmamap_unload(ring->dma_tag, mb_list->dma_map); } - mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); + mb = mlx4_en_alloc_mbuf(ring); if (unlikely(mb == NULL)) goto use_spare; - /* setup correct length */ - mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size; - /* make sure IP header gets aligned */ m_adj(mb, MLX4_NET_IP_ALIGN); @@ -132,7 +171,20 @@ mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, goto use_spare; } - *pdma = cpu_to_be64(segs[0].ds_addr); +#if (MLX4_EN_MAX_RX_SEGS == 1) + rx_desc->data[0].addr = cpu_to_be64(segs[0].ds_addr); +#else + for (i = 0; i != nsegs; i++) { + rx_desc->data[i].byte_count = cpu_to_be32(segs[i].ds_len); + rx_desc->data[i].lkey = ring->rx_mr_key_be; + rx_desc->data[i].addr = cpu_to_be64(segs[i].ds_addr); + } + for (; i != MLX4_EN_MAX_RX_SEGS; i++) { + rx_desc->data[i].byte_count = 0; + rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); + rx_desc->data[i].addr = 0; + } +#endif mb_list->mbuf = mb; bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, BUS_DMASYNC_PREREAD); @@ -149,7 +201,21 @@ use_spare: ring->spare.mbuf = NULL; /* store physical address */ - *pdma = ring->spare.paddr_be; +#if (MLX4_EN_MAX_RX_SEGS == 1) + rx_desc->data[0].addr = cpu_to_be64(ring->spare.segs[0].ds_addr); +#else + for (i = 0; i != MLX4_EN_MAX_RX_SEGS; i++) { + if (ring->spare.segs[i].ds_len != 0) { + rx_desc->data[i].byte_count = cpu_to_be32(ring->spare.segs[i].ds_len); + rx_desc->data[i].lkey = ring->rx_mr_key_be; + rx_desc->data[i].addr = cpu_to_be64(ring->spare.segs[i].ds_addr); + } else { + rx_desc->data[i].byte_count = 0; + rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); + rx_desc->data[i].addr = 0; + } + } +#endif return (0); } @@ -167,13 +233,13 @@ static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, int index) { - struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *) - (ring->buf + (index * ring->stride)); + struct mlx4_en_rx_desc *rx_desc = + ((struct mlx4_en_rx_desc *)ring->buf) + index; struct mlx4_en_rx_mbuf *mb_list = ring->mbuf + index; mb_list->mbuf = NULL; - if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) { + if (mlx4_en_alloc_buf(ring, rx_desc, mb_list)) { priv->port_stats.rx_alloc_failed++; return (-ENOMEM); } @@ -321,7 +387,7 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ MJUM16BYTES, /* maxsize */ - 1, /* nsegments */ + MLX4_EN_MAX_RX_SEGS, /* nsegments */ MJUM16BYTES, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockfuncarg */ @@ -334,11 +400,10 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, ring->cons = 0; ring->size = size; ring->size_mask = size - 1; - ring->stride = roundup_pow_of_two( - sizeof(struct mlx4_en_rx_desc) + DS_SIZE); - ring->log_stride = ffs(ring->stride) - 1; - ring->buf_size = ring->size * ring->stride + TXBB_SIZE; + ring->log_stride = ilog2(sizeof(struct mlx4_en_rx_desc)); + ring->buf_size = (ring->size * sizeof(struct mlx4_en_rx_desc)) + TXBB_SIZE; + tmp = size * sizeof(struct mlx4_en_rx_mbuf); ring->mbuf = kzalloc(tmp, GFP_KERNEL); @@ -398,11 +463,11 @@ err_ring: int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv) { struct mlx4_en_rx_ring *ring; +#if (MLX4_EN_MAX_RX_SEGS == 1) int i; +#endif int ring_ind; int err; - int stride = roundup_pow_of_two( - sizeof(struct mlx4_en_rx_desc) + DS_SIZE); for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { ring = priv->rx_ring[ring_ind]; @@ -413,8 +478,7 @@ int mlx4_en_activate_rx_rings(struct mlx4_en_priv *pri ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn; ring->rx_mb_size = priv->rx_mb_size; - ring->stride = stride; - if (ring->stride <= TXBB_SIZE) { + if (sizeof(struct mlx4_en_rx_desc) <= TXBB_SIZE) { /* Stamp first unused send wqe */ __be32 *ptr = (__be32 *)ring->buf; __be32 stamp = cpu_to_be32(1 << STAMP_SHIFT); @@ -423,15 +487,18 @@ int mlx4_en_activate_rx_rings(struct mlx4_en_priv *pri ring->buf += TXBB_SIZE; } - ring->log_stride = ffs(ring->stride) - 1; - ring->buf_size = ring->size * ring->stride; + ring->log_stride = ilog2(sizeof(struct mlx4_en_rx_desc)); + ring->buf_size = ring->size * sizeof(struct mlx4_en_rx_desc); memset(ring->buf, 0, ring->buf_size); mlx4_en_update_rx_prod_db(ring); +#if (MLX4_EN_MAX_RX_SEGS == 1) /* Initialize all descriptors */ for (i = 0; i < ring->size; i++) mlx4_en_init_rx_desc(priv, ring, i); +#endif + ring->rx_mr_key_be = cpu_to_be32(priv->mdev->mr.key); #ifdef INET /* Configure lro mngr */ @@ -466,7 +533,7 @@ err_buffers: while (ring_ind >= 0) { ring = priv->rx_ring[ring_ind]; - if (ring->stride <= TXBB_SIZE) + if (sizeof(struct mlx4_en_rx_desc) <= TXBB_SIZE) ring->buf -= TXBB_SIZE; ring_ind--; } @@ -477,14 +544,14 @@ err_buffers: void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring **pring, - u32 size, u16 stride) + u32 size) { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_rx_ring *ring = *pring; uint32_t x; mlx4_en_unmap_buffer(&ring->wqres.buf); - mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE); + mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * sizeof(struct mlx4_en_rx_desc) + TXBB_SIZE); for (x = 0; x != size; x++) bus_dmamap_destroy(ring->dma_tag, ring->mbuf[x].dma_map); /* free spare mbuf, if any */ @@ -511,7 +578,7 @@ void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *p tcp_lro_free(&ring->lro); #endif mlx4_en_free_rx_buf(priv, ring); - if (ring->stride <= TXBB_SIZE) + if (sizeof(struct mlx4_en_rx_desc) <= TXBB_SIZE) ring->buf -= TXBB_SIZE; } @@ -557,21 +624,43 @@ mlx4_en_rx_mb(struct mlx4_en_priv *priv, struct mlx4_e struct mlx4_en_rx_desc *rx_desc, struct mlx4_en_rx_mbuf *mb_list, int length) { +#if (MLX4_EN_MAX_RX_SEGS != 1) + struct mbuf *mb_head; +#endif struct mbuf *mb; /* get mbuf */ mb = mb_list->mbuf; /* collect used fragment while atomically replacing it */ - if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) + if (mlx4_en_alloc_buf(ring, rx_desc, mb_list)) return (NULL); /* range check hardware computed value */ - if (unlikely(length > mb->m_len)) - length = mb->m_len; + if (unlikely(length > mb->m_pkthdr.len)) + length = mb->m_pkthdr.len; +#if (MLX4_EN_MAX_RX_SEGS == 1) /* update total packet length in packet header */ mb->m_len = mb->m_pkthdr.len = length; +#else + mb->m_pkthdr.len = length; + for (mb_head = mb; mb != NULL; mb = mb->m_next) { + if (mb->m_len > length) + mb->m_len = length; + length -= mb->m_len; + if (likely(length == 0)) { + if (likely(mb->m_next != NULL)) { + /* trim off empty mbufs */ + m_freem(mb->m_next); + mb->m_next = NULL; + } + break; + } + } + /* rewind to first mbuf in chain */ + mb = mb_head; +#endif return (mb); } @@ -660,8 +749,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, stru while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, cons_index & size)) { mb_list = ring->mbuf + index; - rx_desc = (struct mlx4_en_rx_desc *) - (ring->buf + (index << ring->log_stride)); + rx_desc = ((struct mlx4_en_rx_desc *)ring->buf) + index; /* * make sure we read the CQE after we read the ownership bit @@ -830,7 +918,7 @@ static int mlx4_en_config_rss_qp(struct mlx4_en_priv * qp->event = mlx4_en_sqp_event; memset(context, 0, sizeof *context); - mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0, + mlx4_en_fill_qp_context(priv, ring->actual_size, sizeof(struct mlx4_en_rx_desc), 0, 0, qpn, ring->cqn, -1, context); context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201812121152.wBCBqmJu063379>