Date: Wed, 24 Jun 2026 19:51:22 +0000 From: John Baldwin <jhb@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 45c0d87c5729 - main - OFED: A few channges from Linux 5.0 Message-ID: <6a3c353a.4548f.3fb48026@gitrepo.freebsd.org>
index | next in thread | raw e-mail
The branch main has been updated by jhb: URL: https://cgit.FreeBSD.org/src/commit/?id=45c0d87c57298599397204179c2c4fa0f580a5d9 commit 45c0d87c57298599397204179c2c4fa0f580a5d9 Author: John Baldwin <jhb@FreeBSD.org> AuthorDate: 2026-06-24 19:30:56 +0000 Commit: John Baldwin <jhb@FreeBSD.org> CommitDate: 2026-06-24 19:30:56 +0000 OFED: A few channges from Linux 5.0 The main point of these changes is the last commit adding a couple of helper functions used in ROCE drivers. This contains changes from the following Linux commits: a70c07397fd8 RDMA: Introduce and use GID attr helper to read RoCE L2 fields 8f9748602491 IB/cm: Reduce dependency on gid attribute ndev check adb4a57a7a1d RDMA/cma: Use rdma_read_gid_attr_ndev_rcu to access netdev b4fb4cc5ba83 RDMA/cma: Fix unbalanced cm_id reference count during address resolve d5665a21250e RDMA/core: Add hash functions to calculate RoCEv2 flowlabel and UDP source port Tested by: Wafa Hamzah <wafah@nvidia.com> (mlx5_ib) Tested by: John Baldwin <jhb@FreeBSD.org> (iw_cxgbe) Sponsored by: Chelsio Communications --- sys/dev/bnxt/bnxt_re/ib_verbs.c | 21 ++++++++------ sys/dev/mlx4/mlx4_ib/mlx4_ib_ah.c | 8 ++++-- sys/dev/mlx4/mlx4_ib/mlx4_ib_qp.c | 6 ++-- sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c | 8 ++++-- sys/ofed/drivers/infiniband/core/ib_addr.c | 1 + sys/ofed/drivers/infiniband/core/ib_cache.c | 32 +++++++++++++++++++++ sys/ofed/drivers/infiniband/core/ib_cm.c | 5 ++-- sys/ofed/drivers/infiniband/core/ib_cma.c | 14 +++++++-- sys/ofed/include/rdma/ib_cache.h | 4 +++ sys/ofed/include/rdma/ib_verbs.h | 44 +++++++++++++++++++++++++++++ 10 files changed, 123 insertions(+), 20 deletions(-) diff --git a/sys/dev/bnxt/bnxt_re/ib_verbs.c b/sys/dev/bnxt/bnxt_re/ib_verbs.c index 0162c130d23e..48be755b78a0 100644 --- a/sys/dev/bnxt/bnxt_re/ib_verbs.c +++ b/sys/dev/bnxt/bnxt_re/ib_verbs.c @@ -517,8 +517,10 @@ int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context) struct bnxt_re_gid_ctx *ctx, **ctx_tbl; struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev); struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; - if ((attr->ndev) && is_vlan_dev(attr->ndev)) - vlan_id = vlan_dev_vlan_id(attr->ndev); + + rc = rdma_read_gid_l2_fields(attr, &vlan_id, NULL); + if (rc) + return rc; rc = bnxt_qplib_add_sgid(sgid_tbl, &attr->gid, rdev->dev_addr, @@ -936,13 +938,11 @@ static int bnxt_re_get_ah_info(struct bnxt_re_dev *rdev, ah_info->sgid = gattr->gid; /* Get vlan tag */ - if (gattr->ndev) { - if (is_vlan_dev(gattr->ndev)) - ah_info->vlan_tag = vlan_dev_vlan_id(gattr->ndev); - } + rc = rdma_read_gid_l2_fields(gattr, &ah_info->vlan_tag, NULL); + if (rc) + return rc; /* Get network header type for this GID */ - ib_ntype = rdma_gid_attr_network_type(gattr); ntype = _to_bnxt_re_nw_type(ib_ntype); ah_info->nw_type = ntype; @@ -2649,8 +2649,11 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, sgid_attr = grh->sgid_attr; gid_ptr = &sgid_attr->gid; if (sgid_attr->ndev) { - memcpy(qp->qplib_qp.smac, rdev->dev_addr, - ETH_ALEN); + rc = rdma_read_gid_l2_fields(sgid_attr, NULL, + &qp->qplib_qp.smac[0]); + if (rc) + return rc; + nw_type = rdma_gid_attr_network_type(sgid_attr); dev_dbg(rdev_to_dev(rdev), "Connection using the nw_type %d\n", nw_type); diff --git a/sys/dev/mlx4/mlx4_ib/mlx4_ib_ah.c b/sys/dev/mlx4/mlx4_ib/mlx4_ib_ah.c index babdd234e45b..4ae31c59c2cf 100644 --- a/sys/dev/mlx4/mlx4_ib/mlx4_ib_ah.c +++ b/sys/dev/mlx4/mlx4_ib/mlx4_ib_ah.c @@ -95,9 +95,11 @@ static int create_iboe_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr) memcpy(ah->av.eth.mac, ah_attr->roce.dmac, ETH_ALEN); eth_zero_addr(ah->av.eth.s_mac); gid_attr = ah_attr->grh.sgid_attr; - vlan_tag = rdma_vlan_dev_vlan_id(gid_attr->ndev); - memcpy(ah->av.eth.s_mac, if_getlladdr(gid_attr->ndev), ETH_ALEN); - + ret = rdma_read_gid_l2_fields(gid_attr, &vlan_tag, + &ah->av.eth.s_mac[0]); + if (ret) + return ret; + if (vlan_tag < 0x1000) vlan_tag |= (rdma_ah_get_sl(ah_attr) & 7) << 13; ah->av.eth.port_pd = cpu_to_be32(to_mpd(pd)->pdn | diff --git a/sys/dev/mlx4/mlx4_ib/mlx4_ib_qp.c b/sys/dev/mlx4/mlx4_ib/mlx4_ib_qp.c index fb98bdcd48f9..a411bd2c48ee 100644 --- a/sys/dev/mlx4/mlx4_ib/mlx4_ib_qp.c +++ b/sys/dev/mlx4/mlx4_ib/mlx4_ib_qp.c @@ -1780,8 +1780,10 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp, if (is_eth) { gid_attr = attr->ah_attr.grh.sgid_attr; - vlan = rdma_vlan_dev_vlan_id(gid_attr->ndev); - memcpy(smac, if_getlladdr(gid_attr->ndev), ETH_ALEN); + err = rdma_read_gid_l2_fields(gid_attr, &vlan, + &smac[0]); + if (err) + goto out; } if (mlx4_set_path(dev, attr, attr_mask, qp, &context->pri_path, diff --git a/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c b/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c index 90b886d635f1..53ccf68e4077 100644 --- a/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c +++ b/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c @@ -368,12 +368,16 @@ static void ib_gid_to_mlx5_roce_addr(const union ib_gid *gid, void *mlx5_addr_mac = MLX5_ADDR_OF(roce_addr_layout, mlx5_addr, source_mac_47_32); u16 vlan_id; + int ret; if (!gid) return; - ether_addr_copy(mlx5_addr_mac, if_getlladdr(attr->ndev)); - vlan_id = rdma_vlan_dev_vlan_id(attr->ndev); + + ret = rdma_read_gid_l2_fields(attr, &vlan_id, mlx5_addr_mac); + if (ret != 0) + return; + if (vlan_id != 0xffff) { MLX5_SET_RA(mlx5_addr, vlan_valid, 1); MLX5_SET_RA(mlx5_addr, vlan_id, vlan_id); diff --git a/sys/ofed/drivers/infiniband/core/ib_addr.c b/sys/ofed/drivers/infiniband/core/ib_addr.c index 87370f4aba99..8a18f7bab1d2 100644 --- a/sys/ofed/drivers/infiniband/core/ib_addr.c +++ b/sys/ofed/drivers/infiniband/core/ib_addr.c @@ -45,6 +45,7 @@ #include <net/route/nhop.h> #include <net/netevent.h> #include <rdma/ib_addr.h> +#include <rdma/ib_cache.h> #include <rdma/ib_sa.h> #include <rdma/ib.h> diff --git a/sys/ofed/drivers/infiniband/core/ib_cache.c b/sys/ofed/drivers/infiniband/core/ib_cache.c index 1ad577ff4b83..e5910d91673b 100644 --- a/sys/ofed/drivers/infiniband/core/ib_cache.c +++ b/sys/ofed/drivers/infiniband/core/ib_cache.c @@ -38,6 +38,7 @@ #include <sys/cdefs.h> #include <linux/module.h> #include <linux/errno.h> +#include <linux/etherdevice.h> #include <linux/slab.h> #include <linux/workqueue.h> #include <linux/netdevice.h> @@ -1297,6 +1298,37 @@ if_t rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr) read_unlock_irqrestore(&table->rwlock, flags); return ndev; } +EXPORT_SYMBOL(rdma_read_gid_attr_ndev_rcu); + +/** + * rdma_read_gid_l2_fields - Read the vlan ID and source MAC address + * of a GID entry. + * + * @attr: GID attribute pointer whose L2 fields to be read + * @vlan_id: Pointer to vlan id to fill up if the GID entry has + * vlan id. It is optional. + * @smac: Pointer to smac to fill up for a GID entry. It is optional. + * + * rdma_read_gid_l2_fields() returns 0 on success and returns vlan id + * (if gid entry has vlan) and source MAC, or returns error. + */ +int rdma_read_gid_l2_fields(const struct ib_gid_attr *attr, + u16 *vlan_id, u8 *smac) +{ + if_t ndev; + + ndev = attr->ndev; + if (!ndev) + return -EINVAL; + + if (smac) + ether_addr_copy(smac, if_getlladdr(ndev)); + if (vlan_id) { + *vlan_id = rdma_vlan_dev_vlan_id(ndev); + } + return 0; +} +EXPORT_SYMBOL(rdma_read_gid_l2_fields); static int config_non_roce_gid_cache(struct ib_device *device, u8 port, int gid_tbl_len) diff --git a/sys/ofed/drivers/infiniband/core/ib_cm.c b/sys/ofed/drivers/infiniband/core/ib_cm.c index 1ceb08ce2d61..d2377b343a57 100644 --- a/sys/ofed/drivers/infiniband/core/ib_cm.c +++ b/sys/ofed/drivers/infiniband/core/ib_cm.c @@ -1985,11 +1985,12 @@ static int cm_req_handler(struct cm_work *work) grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr); gid_attr = grh->sgid_attr; - if (gid_attr && gid_attr->ndev) { + if (gid_attr && + rdma_protocol_roce(work->port->cm_dev->ib_device, + work->port->port_num)) { work->path[0].rec_type = sa_conv_gid_to_pathrec_type(gid_attr->gid_type); } else { - /* If no GID attribute or ndev is null, it is not RoCE. */ cm_path_set_rec_type(work->port->cm_dev->ib_device, work->port->port_num, &work->path[0], diff --git a/sys/ofed/drivers/infiniband/core/ib_cma.c b/sys/ofed/drivers/infiniband/core/ib_cma.c index a16e31df6752..ea2e04309a47 100644 --- a/sys/ofed/drivers/infiniband/core/ib_cma.c +++ b/sys/ofed/drivers/infiniband/core/ib_cma.c @@ -1537,6 +1537,7 @@ static if_t roce_get_net_dev_by_cm_event(const struct ib_cm_event *ib_event) { const struct ib_gid_attr *sgid_attr = NULL; + if_t ndev; if (ib_event->event == IB_CM_REQ_RECEIVED) sgid_attr = ib_event->param.req_rcvd.ppath_sgid_attr; @@ -1545,8 +1546,15 @@ roce_get_net_dev_by_cm_event(const struct ib_cm_event *ib_event) if (!sgid_attr) return NULL; - dev_hold(sgid_attr->ndev); - return sgid_attr->ndev; + + rcu_read_lock(); + ndev = rdma_read_gid_attr_ndev_rcu(sgid_attr); + if (IS_ERR(ndev)) + ndev = NULL; + else + dev_hold(ndev); + rcu_read_unlock(); + return ndev; } static if_t cma_get_net_dev(const struct ib_cm_event *ib_event, @@ -3095,6 +3103,7 @@ static int cma_resolve_loopback(struct rdma_id_private *id_priv) rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid); rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid); + atomic_inc(&id_priv->refcount); cma_init_resolve_addr_work(work, id_priv); queue_work(cma_wq, &work->work); return 0; @@ -3121,6 +3130,7 @@ static int cma_resolve_ib_addr(struct rdma_id_private *id_priv) rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, (union ib_gid *) &(((struct sockaddr_ib *) &id_priv->id.route.addr.dst_addr)->sib_addr)); + atomic_inc(&id_priv->refcount); cma_init_resolve_addr_work(work, id_priv); queue_work(cma_wq, &work->work); return 0; diff --git a/sys/ofed/include/rdma/ib_cache.h b/sys/ofed/include/rdma/ib_cache.h index cea05d381b44..881b8870f5fc 100644 --- a/sys/ofed/include/rdma/ib_cache.h +++ b/sys/ofed/include/rdma/ib_cache.h @@ -56,6 +56,10 @@ const struct ib_gid_attr *rdma_find_gid_by_filter( void *), void *context); +int rdma_read_gid_l2_fields(const struct ib_gid_attr *attr, + u16 *vlan_id, u8 *smac); +if_t rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr); + /** * ib_get_cached_pkey - Returns a cached PKey table entry * @device: The device to query. diff --git a/sys/ofed/include/rdma/ib_verbs.h b/sys/ofed/include/rdma/ib_verbs.h index b8981913aa24..38e6858e767f 100644 --- a/sys/ofed/include/rdma/ib_verbs.h +++ b/sys/ofed/include/rdma/ib_verbs.h @@ -4179,4 +4179,48 @@ static inline __be16 ib_lid_be16(u32 lid) WARN_ON_ONCE(lid & 0xFFFF0000); return cpu_to_be16((u16)lid); } + +#define IB_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000) +#define IB_GRH_FLOWLABEL_MASK (0x000FFFFF) + +/** + * rdma_flow_label_to_udp_sport - generate a RoCE v2 UDP src port value based + * on the flow_label + * + * This function will convert the 20 bit flow_label input to a valid RoCE v2 + * UDP src port 14 bit value. All RoCE V2 drivers should use this same + * convention. + */ +static inline u16 rdma_flow_label_to_udp_sport(u32 fl) +{ + u32 fl_low = fl & 0x03fff, fl_high = fl & 0xFC000; + + fl_low ^= fl_high >> 14; + return (u16)(fl_low | IB_ROCE_UDP_ENCAP_VALID_PORT_MIN); +} + +/** + * rdma_calc_flow_label - generate a RDMA symmetric flow label value based on + * local and remote qpn values + * + * This function folded the multiplication results of two qpns, 24 bit each, + * fields, and converts it to a 20 bit results. + * + * This function will create symmetric flow_label value based on the local + * and remote qpn values. this will allow both the requester and responder + * to calculate the same flow_label for a given connection. + * + * This helper function should be used by driver in case the upper layer + * provide a zero flow_label value. This is to improve entropy of RDMA + * traffic in the network. + */ +static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn) +{ + u64 v = (u64)lqpn * rqpn; + + v ^= v >> 20; + v ^= v >> 40; + + return (u32)(v & IB_GRH_FLOWLABEL_MASK); +} #endif /* IB_VERBS_H */home | help
Want to link to this message? Use this
URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?6a3c353a.4548f.3fb48026>