Date: Fri, 18 Aug 2023 18:30:44 GMT From: Eric Joyner <erj@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 01fbb86991d5 - main - irdma(4): upgrade to 1.2.17-k Message-ID: <202308181830.37IIUio7046074@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by erj: URL: https://cgit.FreeBSD.org/src/commit/?id=01fbb86991d524288a785b544c29d9c5ea1b61fb commit 01fbb86991d524288a785b544c29d9c5ea1b61fb Author: Bartosz Sobczak <bartosz.sobczak@intel.com> AuthorDate: 2023-08-18 18:14:33 +0000 Commit: Eric Joyner <erj@FreeBSD.org> CommitDate: 2023-08-18 18:28:58 +0000 irdma(4): upgrade to 1.2.17-k Update Intel irdma driver to version 1.2.17-k Notable changes: - pf-reset handling improvements, including fixes in communication with if_ice(4) - avoid racing when handling various events - adding sw stats sysctls - hand over pe_criterr handling from ice(4) to irdma(4) - debug prints adjustments - fix crash after changes in irdma_add_mqh_ifa_cb Signed-off-by: Bartosz Sobczak <bartosz.sobczak@intel.com> Signed-off-by: Eric Joyner <erj@FreeBSD.org> Reviewed by: erj@ MFC after: 1 month Sponsored by: Intel Corporation Differential Revision: https://reviews.freebsd.org/D41425 --- contrib/ofed/libirdma/irdma_defs.h | 25 +- contrib/ofed/libirdma/irdma_uk.c | 59 +---- contrib/ofed/libirdma/irdma_umain.c | 4 +- contrib/ofed/libirdma/irdma_user.h | 3 +- contrib/ofed/libirdma/irdma_uverbs.c | 61 ++--- contrib/ofed/libirdma/osdep.h | 82 +++---- sys/dev/ice/ice_rdma.c | 59 ++++- sys/dev/ice/ice_rdma.h | 11 +- sys/dev/ice/ice_rdma_internal.h | 2 + sys/dev/ice/if_ice_iflib.c | 15 +- sys/dev/irdma/fbsd_kcompat.c | 282 ++++++++++++++++++++++- sys/dev/irdma/fbsd_kcompat.h | 16 +- sys/dev/irdma/icrdma.c | 283 +++++++++++++++-------- sys/dev/irdma/irdma.h | 4 +- sys/dev/irdma/irdma_cm.c | 303 ++++++++++++------------ sys/dev/irdma/irdma_cm.h | 15 +- sys/dev/irdma/irdma_ctrl.c | 160 +++++-------- sys/dev/irdma/irdma_defs.h | 87 +++---- sys/dev/irdma/irdma_hmc.c | 6 +- sys/dev/irdma/irdma_hmc.h | 6 +- sys/dev/irdma/irdma_hw.c | 99 ++++---- sys/dev/irdma/irdma_kcompat.c | 121 +++++----- sys/dev/irdma/irdma_main.h | 12 +- sys/dev/irdma/irdma_pble.c | 16 +- sys/dev/irdma/irdma_pble.h | 6 +- sys/dev/irdma/irdma_protos.h | 8 +- sys/dev/irdma/irdma_puda.c | 8 +- sys/dev/irdma/irdma_puda.h | 6 +- sys/dev/irdma/irdma_type.h | 21 +- sys/dev/irdma/irdma_uda.c | 17 +- sys/dev/irdma/irdma_uda.h | 4 +- sys/dev/irdma/irdma_uk.c | 84 +------ sys/dev/irdma/irdma_user.h | 3 +- sys/dev/irdma/irdma_utils.c | 230 ++++++++++++++----- sys/dev/irdma/irdma_verbs.c | 431 +++++++++++++++++++---------------- sys/dev/irdma/irdma_verbs.h | 91 ++++++-- sys/dev/irdma/irdma_ws.c | 4 +- sys/dev/irdma/osdep.h | 61 +++-- 38 files changed, 1586 insertions(+), 1119 deletions(-) diff --git a/contrib/ofed/libirdma/irdma_defs.h b/contrib/ofed/libirdma/irdma_defs.h index 3d8b59c4b78e..9276490793a0 100644 --- a/contrib/ofed/libirdma/irdma_defs.h +++ b/contrib/ofed/libirdma/irdma_defs.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -312,7 +312,7 @@ #define IRDMA_GET_CQ_ELEM_AT_OFFSET(_cq, _i, _cqe) \ { \ - register __u32 offset; \ + __u32 offset; \ offset = IRDMA_GET_RING_OFFSET((_cq)->cq_ring, _i); \ (_cqe) = (_cq)->cq_base[offset].buf; \ } @@ -338,7 +338,7 @@ #define IRDMA_RING_MOVE_HEAD(_ring, _retcode) \ { \ - register u32 size; \ + u32 size; \ size = (_ring).size; \ if (!IRDMA_RING_FULL_ERR(_ring)) { \ (_ring).head = ((_ring).head + 1) % size; \ @@ -349,7 +349,7 @@ } #define IRDMA_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \ { \ - register u32 size; \ + u32 size; \ size = (_ring).size; \ if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < size) { \ (_ring).head = ((_ring).head + (_count)) % size; \ @@ -360,7 +360,7 @@ } #define IRDMA_SQ_RING_MOVE_HEAD(_ring, _retcode) \ { \ - register u32 size; \ + u32 size; \ size = (_ring).size; \ if (!IRDMA_SQ_RING_FULL_ERR(_ring)) { \ (_ring).head = ((_ring).head + 1) % size; \ @@ -371,7 +371,7 @@ } #define IRDMA_SQ_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \ { \ - register u32 size; \ + u32 size; \ size = (_ring).size; \ if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < (size - 256)) { \ (_ring).head = ((_ring).head + (_count)) % size; \ @@ -457,6 +457,19 @@ enum irdma_qp_wqe_size { IRDMA_WQE_SIZE_256 = 256, }; +enum irdma_ws_op_type { + IRDMA_WS_OP_TYPE_NODE = 0, + IRDMA_WS_OP_TYPE_LEAF_NODE_GROUP, +}; + +enum irdma_ws_rate_limit_flags { + IRDMA_WS_RATE_LIMIT_FLAGS_VALID = 0x1, + IRDMA_WS_NO_RDMA_RATE_LIMIT = 0x2, + IRDMA_WS_LEAF_NODE_IS_PART_GROUP = 0x4, + IRDMA_WS_TREE_RATE_LIMITING = 0x8, + IRDMA_WS_PACING_CONTROL = 0x10, +}; + /** * set_64bit_val - set 64 bit value to hw wqe * @wqe_words: wqe addr to write diff --git a/contrib/ofed/libirdma/irdma_uk.c b/contrib/ofed/libirdma/irdma_uk.c index 97e3ac553c26..a1b796b318f4 100644 --- a/contrib/ofed/libirdma/irdma_uk.c +++ b/contrib/ofed/libirdma/irdma_uk.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -275,7 +275,8 @@ irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx, if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && wqe_quanta == 1 && (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) { wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem; - wqe_0[3] = htole64(FIELD_PREP(IRDMAQPSQ_VALID, !qp->swqe_polarity)); + wqe_0[3] = htole64(FIELD_PREP(IRDMAQPSQ_VALID, + qp->swqe_polarity ? 0 : 1)); } qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id; qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size; @@ -683,8 +684,8 @@ irdma_set_mw_bind_wqe(__le64 * wqe, * @polarity: polarity of wqe valid bit */ static void -irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list, u32 num_sges, - u8 polarity) +irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list, + u32 num_sges, u8 polarity) { u8 inline_valid = polarity << IRDMA_INLINE_VALID_S; u32 quanta_bytes_remaining = 8; @@ -1173,7 +1174,7 @@ irdma_repost_rq_wqes(struct irdma_qp_uk *qp, u32 start_idx, u32 end_idx) { __le64 *dst_wqe, *src_wqe; - u32 wqe_idx; + u32 wqe_idx = 0; u8 wqe_quanta = qp->rq_wqe_size_multiplier; bool flip_polarity; u64 val; @@ -1480,7 +1481,8 @@ irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, sw_wqe = qp->sq_base[tail].elem; get_64bit_val(sw_wqe, IRDMA_BYTE_24, &wqe_qword); - info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword); + info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, + wqe_qword); IRDMA_RING_SET_TAIL(qp->sq_ring, tail + qp->sq_wrtrk_array[tail].quanta); if (info->op_type != IRDMAQP_OP_NOP) { @@ -1834,6 +1836,9 @@ irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq) if (polarity != temp) break; + /* Ensure CQE contents are read after valid bit is checked */ + udma_from_device_barrier(); + get_64bit_val(cqe, IRDMA_BYTE_8, &comp_ctx); if ((void *)(irdma_uintptr) comp_ctx == q) set_64bit_val(cqe, IRDMA_BYTE_8, 0); @@ -1845,48 +1850,6 @@ irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq) return 0; } -/** - * irdma_nop - post a nop - * @qp: hw qp ptr - * @wr_id: work request id - * @signaled: signaled for completion - * @post_sq: ring doorbell - */ -int -irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq) -{ - __le64 *wqe; - u64 hdr; - u32 wqe_idx; - struct irdma_post_sq_info info = {0}; - u16 quanta = IRDMA_QP_WQE_MIN_QUANTA; - - info.push_wqe = qp->push_db ? true : false; - info.wr_id = wr_id; - wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, &info); - if (!wqe) - return ENOSPC; - - set_64bit_val(wqe, IRDMA_BYTE_0, 0); - set_64bit_val(wqe, IRDMA_BYTE_8, 0); - set_64bit_val(wqe, IRDMA_BYTE_16, 0); - - hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) | - FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) | - FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); - - udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ - - set_64bit_val(wqe, IRDMA_BYTE_24, hdr); - - if (info.push_wqe) - irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); - else if (post_sq) - irdma_uk_qp_post_wr(qp); - - return 0; -} - /** * irdma_fragcnt_to_quanta_sq - calculate quanta based on fragment count for SQ * @frag_cnt: number of fragments diff --git a/contrib/ofed/libirdma/irdma_umain.c b/contrib/ofed/libirdma/irdma_umain.c index 6c823646b375..9e223cae429f 100644 --- a/contrib/ofed/libirdma/irdma_umain.c +++ b/contrib/ofed/libirdma/irdma_umain.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2021 - 2022 Intel Corporation + * Copyright (c) 2021 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -49,7 +49,7 @@ /** * Driver version */ -char libirdma_version[] = "1.1.11-k"; +char libirdma_version[] = "1.2.17-k"; unsigned int irdma_dbg; diff --git a/contrib/ofed/libirdma/irdma_user.h b/contrib/ofed/libirdma/irdma_user.h index 8eb9ae7990e7..8d71a37d7bf4 100644 --- a/contrib/ofed/libirdma/irdma_user.h +++ b/contrib/ofed/libirdma/irdma_user.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -572,7 +572,6 @@ void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge, u32 inline_data, u8 *shift); int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth); int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth); -int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift, u32 *srqdepth); void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta, u32 wqe_idx, bool post_sq); void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx); diff --git a/contrib/ofed/libirdma/irdma_uverbs.c b/contrib/ofed/libirdma/irdma_uverbs.c index 14efab96a107..c59d409e72cd 100644 --- a/contrib/ofed/libirdma/irdma_uverbs.c +++ b/contrib/ofed/libirdma/irdma_uverbs.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (C) 2019 - 2022 Intel Corporation + * Copyright (C) 2019 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -266,11 +266,13 @@ irdma_ualloc_mw(struct ibv_pd *pd, enum ibv_mw_type type) if (!mw) return NULL; - if (ibv_cmd_alloc_mw(pd, type, mw, &cmd, sizeof(cmd), &resp, - sizeof(resp))) { + err = ibv_cmd_alloc_mw(pd, type, mw, &cmd, sizeof(cmd), &resp, + sizeof(resp)); + if (err) { printf("%s: Failed to alloc memory window\n", __func__); free(mw); + errno = err; return NULL; } @@ -446,8 +448,10 @@ ucreate_cq(struct ibv_context *context, if (!iwucq) return NULL; - if (pthread_spin_init(&iwucq->lock, PTHREAD_PROCESS_PRIVATE)) { + ret = pthread_spin_init(&iwucq->lock, PTHREAD_PROCESS_PRIVATE); + if (ret) { free(iwucq); + errno = ret; return NULL; } @@ -464,8 +468,10 @@ ucreate_cq(struct ibv_context *context, iwucq->buf_size = total_size; info.cq_base = irdma_alloc_hw_buf(total_size); - if (!info.cq_base) + if (!info.cq_base) { + ret = ENOMEM; goto err_cq_base; + } memset(info.cq_base, 0, total_size); reg_mr_cmd.reg_type = IRDMA_MEMREG_TYPE_CQ; @@ -476,17 +482,17 @@ ucreate_cq(struct ibv_context *context, IBV_ACCESS_LOCAL_WRITE, &iwucq->vmr.ibv_mr, ®_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd), ®_mr_resp, sizeof(reg_mr_resp)); - if (ret) { - errno = ret; + if (ret) goto err_dereg_mr; - } iwucq->vmr.ibv_mr.pd = &iwvctx->iwupd->ibv_pd; if (uk_attrs->feature_flags & IRDMA_FEATURE_CQ_RESIZE) { info.shadow_area = irdma_alloc_hw_buf(IRDMA_DB_SHADOW_AREA_SIZE); - if (!info.shadow_area) + if (!info.shadow_area) { + ret = ENOMEM; goto err_alloc_shadow; + } memset(info.shadow_area, 0, IRDMA_DB_SHADOW_AREA_SIZE); reg_mr_shadow_cmd.reg_type = IRDMA_MEMREG_TYPE_CQ; @@ -499,7 +505,6 @@ ucreate_cq(struct ibv_context *context, ®_mr_shadow_resp, sizeof(reg_mr_shadow_resp)); if (ret) { irdma_free_hw_buf(info.shadow_area, IRDMA_DB_SHADOW_AREA_SIZE); - errno = ret; goto err_alloc_shadow; } @@ -517,10 +522,8 @@ ucreate_cq(struct ibv_context *context, &cmd.ibv_cmd, sizeof(cmd.ibv_cmd), sizeof(cmd), &resp.ibv_resp, sizeof(resp.ibv_resp), sizeof(resp)); attr_ex->cqe = ncqe; - if (ret) { - errno = ret; + if (ret) goto err_create_cq; - } if (ext_cq) irdma_ibvcq_ex_fill_priv_funcs(iwucq, attr_ex); @@ -548,6 +551,7 @@ err_cq_base: free(iwucq); + errno = ret; return NULL; } @@ -1560,7 +1564,8 @@ irdma_ucreate_qp(struct ibv_pd *pd, memset(iwuqp, 0, sizeof(*iwuqp)); - if (pthread_spin_init(&iwuqp->lock, PTHREAD_PROCESS_PRIVATE)) + status = pthread_spin_init(&iwuqp->lock, PTHREAD_PROCESS_PRIVATE); + if (status) goto err_free_qp; info.sq_size = info.sq_depth >> info.sq_shift; @@ -1575,35 +1580,37 @@ irdma_ucreate_qp(struct ibv_pd *pd, } iwuqp->recv_sges = calloc(attr->cap.max_recv_sge, sizeof(*iwuqp->recv_sges)); - if (!iwuqp->recv_sges) + if (!iwuqp->recv_sges) { + status = errno; /* preserve errno */ goto err_destroy_lock; + } info.wqe_alloc_db = (u32 *)iwvctx->db; info.legacy_mode = iwvctx->legacy_mode; info.sq_wrtrk_array = calloc(info.sq_depth, sizeof(*info.sq_wrtrk_array)); - if (!info.sq_wrtrk_array) + if (!info.sq_wrtrk_array) { + status = errno; /* preserve errno */ goto err_free_rsges; + } info.rq_wrid_array = calloc(info.rq_depth, sizeof(*info.rq_wrid_array)); - if (!info.rq_wrid_array) + if (!info.rq_wrid_array) { + status = errno; /* preserve errno */ goto err_free_sq_wrtrk; + } iwuqp->sq_sig_all = attr->sq_sig_all; iwuqp->qp_type = attr->qp_type; status = irdma_vmapped_qp(iwuqp, pd, attr, &info, iwvctx->legacy_mode); - if (status) { - errno = status; + if (status) goto err_free_rq_wrid; - } iwuqp->qp.back_qp = iwuqp; iwuqp->qp.lock = &iwuqp->lock; status = irdma_uk_qp_init(&iwuqp->qp, &info); - if (status) { - errno = status; + if (status) goto err_free_vmap_qp; - } attr->cap.max_send_wr = (info.sq_depth - IRDMA_SQ_RSVD) >> info.sq_shift; attr->cap.max_recv_wr = (info.rq_depth - IRDMA_RQ_RSVD) >> info.rq_shift; @@ -1625,6 +1632,7 @@ err_free_qp: printf("%s: failed to create QP\n", __func__); free(iwuqp); + errno = status; return NULL; } @@ -2081,11 +2089,10 @@ irdma_ucreate_ah(struct ibv_pd *ibpd, struct ibv_ah_attr *attr) struct irdma_ucreate_ah_resp resp = {}; int err; - err = ibv_query_gid(ibpd->context, attr->port_num, attr->grh.sgid_index, - &sgid); - if (err) { + if (ibv_query_gid(ibpd->context, attr->port_num, attr->grh.sgid_index, + &sgid)) { fprintf(stderr, "irdma: Error from ibv_query_gid.\n"); - errno = err; + errno = ENOENT; return NULL; } diff --git a/contrib/ofed/libirdma/osdep.h b/contrib/ofed/libirdma/osdep.h index 1bbf6573b800..f7b8a96b93d2 100644 --- a/contrib/ofed/libirdma/osdep.h +++ b/contrib/ofed/libirdma/osdep.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2021 - 2022 Intel Corporation + * Copyright (c) 2021 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -46,7 +46,6 @@ #include <sys/bus_dma.h> #include <sys/endian.h> -#define ATOMIC atomic_t #define IOMEM #define IRDMA_NTOHL(a) ntohl(a) #define IRDMA_NTOHS(a) ntohs(a) @@ -80,23 +79,13 @@ #define STATS_TIMER_DELAY 60000 /* a couple of linux size defines */ -#define SZ_128 128 -#define SZ_2K SZ_128 * 16 -#define SZ_1G (SZ_1K * SZ_1K * SZ_1K) -#define SPEED_1000 1000 -#define SPEED_10000 10000 -#define SPEED_20000 20000 -#define SPEED_25000 25000 -#define SPEED_40000 40000 -#define SPEED_100000 100000 - #define BIT_ULL(a) (1ULL << (a)) #define min(a, b) ((a) > (b) ? (b) : (a)) #ifndef likely -#define likely(x) __builtin_expect((x), 1) +#define likely(x) __builtin_expect((x), 1) #endif #ifndef unlikely -#define unlikely(x) __builtin_expect((x), 0) +#define unlikely(x) __builtin_expect((x), 0) #endif #define __aligned_u64 uint64_t __aligned(8) @@ -112,7 +101,7 @@ #define irdma_print(S, ...) printf("%s:%d "S, __FUNCTION__, __LINE__, ##__VA_ARGS__) #define irdma_debug_buf(dev, mask, desc, buf, size) \ do { \ - u32 i; \ + u32 i; \ if (!((mask) & (dev)->debug_mask)) { \ break; \ } \ @@ -122,20 +111,20 @@ do { \ irdma_debug(dev, mask, "index %03d val: %016lx\n", i, ((unsigned long *)(buf))[i / 8]); \ } while(0) -#define irdma_debug(h, m, s, ...) \ -do { \ - if (!(h)) { \ - if ((m) == IRDMA_DEBUG_INIT) \ +#define irdma_debug(h, m, s, ...) \ +do { \ + if (!(h)) { \ + if ((m) == IRDMA_DEBUG_INIT) \ printf("irdma INIT " s, ##__VA_ARGS__); \ - } else if (((m) & (h)->debug_mask)) { \ - printf("irdma " s, ##__VA_ARGS__); \ - } \ + } else if (((m) & (h)->debug_mask)) { \ + printf("irdma " s, ##__VA_ARGS__); \ + } \ } while (0) extern unsigned int irdma_dbg; -#define libirdma_debug(fmt, args...) \ -do { \ - if (irdma_dbg) \ - printf("libirdma-%s: " fmt, __func__, ##args); \ +#define libirdma_debug(fmt, args...) \ +do { \ + if (irdma_dbg) \ + printf("libirdma-%s: " fmt, __func__, ##args); \ } while (0) #define irdma_dev_err(ibdev, fmt, ...) \ pr_err("%s:%s:%d ERR "fmt, (ibdev)->name, __func__, __LINE__, ##__VA_ARGS__) @@ -144,18 +133,6 @@ do { \ #define irdma_dev_info(a, b, ...) printf(b, ##__VA_ARGS__) #define irdma_pr_warn printf -#define dump_struct(s, sz, name) \ -do { \ - unsigned char *a; \ - printf("%s %u", (name), (unsigned int)(sz)); \ - for (a = (unsigned char*)(s); a < (unsigned char *)(s) + (sz) ; a ++) { \ - if ((u64)a % 8 == 0) \ - printf("\n%p ", a); \ - printf("%2x ", *a); \ - } \ - printf("\n"); \ -}while(0) - /* * debug definition end */ @@ -179,13 +156,6 @@ struct irdma_sc_vsi; #define rt_tos2priority(tos) (tos >> 5) #define ah_attr_to_dmac(attr) ((attr).dmac) -#define kc_ib_modify_qp_is_ok(cur_state, next_state, type, mask, ll) \ - ib_modify_qp_is_ok(cur_state, next_state, type, mask) -#define kc_typeq_ib_wr const -#define kc_ifp_find ip_ifp_find -#define kc_ifp6_find ip6_ifp_find -#define kc_rdma_gid_attr_network_type(sgid_attr, gid_type, gid) \ - ib_gid_to_network_type(gid_type, gid) #define irdma_del_timer_compat(tt) del_timer((tt)) #define IRDMA_TAILQ_FOREACH CK_STAILQ_FOREACH #define IRDMA_TAILQ_FOREACH_SAFE CK_STAILQ_FOREACH_SAFE @@ -193,36 +163,36 @@ struct irdma_sc_vsi; static inline void db_wr32(__u32 val, __u32 *wqe_word) { - *wqe_word = val; + *wqe_word = val; } void *hw_to_dev(struct irdma_hw *hw); struct irdma_dma_mem { - void *va; - u64 pa; + void *va; + u64 pa; bus_dma_tag_t tag; bus_dmamap_t map; bus_dma_segment_t seg; bus_size_t size; - int nseg; - int flags; + int nseg; + int flags; }; struct irdma_virt_mem { - void *va; - u32 size; + void *va; + u32 size; }; #ifndef verbs_mr enum ibv_mr_type { - IBV_MR_TYPE_MR, - IBV_MR_TYPE_NULL_MR, + IBV_MR_TYPE_MR, + IBV_MR_TYPE_NULL_MR, }; struct verbs_mr { - struct ibv_mr ibv_mr; - enum ibv_mr_type mr_type; + struct ibv_mr ibv_mr; + enum ibv_mr_type mr_type; int access; }; #define verbs_get_mr(mr) container_of((mr), struct verbs_mr, ibv_mr) diff --git a/sys/dev/ice/ice_rdma.c b/sys/dev/ice/ice_rdma.c index decf9ea490d8..0bb4b79bb143 100644 --- a/sys/dev/ice/ice_rdma.c +++ b/sys/dev/ice/ice_rdma.c @@ -121,11 +121,10 @@ ice_rdma_pf_reset(struct ice_rdma_peer *peer) { struct ice_softc *sc = ice_rdma_peer_to_sc(peer); - /* - * Request that the driver re-initialize by bringing the interface - * down and up. - */ - ice_request_stack_reinit(sc); + /* Tell the base driver that RDMA is requesting a PFR */ + ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); + + /* XXX: Base driver will notify RDMA when it's done */ return (0); } @@ -331,6 +330,7 @@ ice_rdma_request_handler(struct ice_rdma_peer *peer, switch(req->type) { case ICE_RDMA_EVENT_RESET: + ice_rdma_pf_reset(peer); break; case ICE_RDMA_EVENT_QSET_REGISTER: ice_rdma_qset_register_request(peer, &req->res); @@ -865,3 +865,52 @@ ice_rdma_dcb_qos_update(struct ice_softc *sc, struct ice_port_info *pi) IRDMA_EVENT_HANDLER(peer, &event); sx_xunlock(&ice_rdma.mtx); } + +/** + * ice_rdma_notify_pe_intr - notify irdma on incoming interrupts regarding PE + * @sc: the ice driver softc + * @oicr: interrupt cause + * + * Pass the information about received interrupt to RDMA driver if it was + * relating to PE. Specifically PE_CRITERR and HMC_ERR. + * The irdma driver shall decide what should be done upon these interrupts. + */ +void +ice_rdma_notify_pe_intr(struct ice_softc *sc, uint32_t oicr) +{ + struct ice_rdma_peer *peer = &sc->rdma_entry.peer; + struct ice_rdma_event event; + + memset(&event, 0, sizeof(struct ice_rdma_event)); + event.type = ICE_RDMA_EVENT_CRIT_ERR; + event.oicr_reg = oicr; + + sx_xlock(&ice_rdma.mtx); + if (sc->rdma_entry.attached && ice_rdma.registered) + IRDMA_EVENT_HANDLER(peer, &event); + sx_xunlock(&ice_rdma.mtx); +} + +/** + * ice_rdma_notify_reset - notify irdma on incoming pf-reset + * @sc: the ice driver softc + * + * Inform irdma driver of an incoming PF reset. + * The irdma driver shall set its state to reset, and avoid using CQP + * anymore. Next step should be to call ice_rdma_pf_stop in order to + * remove resources. + */ +void +ice_rdma_notify_reset(struct ice_softc *sc) +{ + struct ice_rdma_peer *peer = &sc->rdma_entry.peer; + struct ice_rdma_event event; + + memset(&event, 0, sizeof(struct ice_rdma_event)); + event.type = ICE_RDMA_EVENT_RESET; + + sx_xlock(&ice_rdma.mtx); + if (sc->rdma_entry.attached && ice_rdma.registered) + IRDMA_EVENT_HANDLER(peer, &event); + sx_xunlock(&ice_rdma.mtx); +} diff --git a/sys/dev/ice/ice_rdma.h b/sys/dev/ice/ice_rdma.h index f83c30b33f6c..38e2ef491e8e 100644 --- a/sys/dev/ice/ice_rdma.h +++ b/sys/dev/ice/ice_rdma.h @@ -64,7 +64,7 @@ * considered stable. */ #define ICE_RDMA_MAJOR_VERSION 1 -#define ICE_RDMA_MINOR_VERSION 0 +#define ICE_RDMA_MINOR_VERSION 1 #define ICE_RDMA_PATCH_VERSION 0 /** @@ -273,18 +273,19 @@ struct ice_rdma_event { uint64_t baudrate; }; /* MTU change event */ - struct { - int mtu; - }; + int mtu; /* * TC/QoS/DCB change event - * RESET event use prep variable only * prep: if true, this is a pre-event, post-event otherwise */ struct { struct ice_qos_params port_qos; bool prep; }; + /* + * CRIT_ERR event + */ + uint32_t oicr_reg; }; }; diff --git a/sys/dev/ice/ice_rdma_internal.h b/sys/dev/ice/ice_rdma_internal.h index c80d4540194a..b36544609f45 100644 --- a/sys/dev/ice/ice_rdma_internal.h +++ b/sys/dev/ice/ice_rdma_internal.h @@ -98,4 +98,6 @@ int ice_rdma_pf_stop(struct ice_softc *sc); void ice_rdma_link_change(struct ice_softc *sc, int linkstate, uint64_t baudrate); void ice_rdma_notify_dcb_qos_change(struct ice_softc *sc); void ice_rdma_dcb_qos_update(struct ice_softc *sc, struct ice_port_info *pi); +void ice_rdma_notify_pe_intr(struct ice_softc *sc, uint32_t oicr); +void ice_rdma_notify_reset(struct ice_softc *sc); #endif diff --git a/sys/dev/ice/if_ice_iflib.c b/sys/dev/ice/if_ice_iflib.c index cb4931c4d54e..2397fce11596 100644 --- a/sys/dev/ice/if_ice_iflib.c +++ b/sys/dev/ice/if_ice_iflib.c @@ -1278,9 +1278,11 @@ ice_msix_admin(void *arg) ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); } - if (oicr & PFINT_OICR_PE_CRITERR_M) { - device_printf(dev, "Critical Protocol Engine Error detected!\n"); - ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); + if (oicr & (PFINT_OICR_PE_CRITERR_M | PFINT_OICR_HMC_ERR_M)) { + if (oicr & PFINT_OICR_HMC_ERR_M) + /* Log the HMC errors */ + ice_log_hmc_error(hw, dev); + ice_rdma_notify_pe_intr(sc, oicr); } if (oicr & PFINT_OICR_PCI_EXCEPTION_M) { @@ -1288,11 +1290,6 @@ ice_msix_admin(void *arg) ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); } - if (oicr & PFINT_OICR_HMC_ERR_M) { - /* Log the HMC errors, but don't disable the interrupt cause */ - ice_log_hmc_error(hw, dev); - } - return (FILTER_SCHEDULE_THREAD); } @@ -2298,6 +2295,8 @@ ice_prepare_for_reset(struct ice_softc *sc) if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) return; + /* inform the RDMA client */ + ice_rdma_notify_reset(sc); /* stop the RDMA client */ ice_rdma_pf_stop(sc); diff --git a/sys/dev/irdma/fbsd_kcompat.c b/sys/dev/irdma/fbsd_kcompat.c index 325429cdd993..013f9279952e 100644 --- a/sys/dev/irdma/fbsd_kcompat.c +++ b/sys/dev/irdma/fbsd_kcompat.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2021 - 2022 Intel Corporation + * Copyright (c) 2021 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -273,12 +273,15 @@ irdma_add_ip(struct irdma_device *iwdev) { struct ifnet *ifp = iwdev->netdev; struct ifnet *ifv; + struct epoch_tracker et; int i; irdma_add_ipv4_addr(iwdev, ifp); irdma_add_ipv6_addr(iwdev, ifp); for (i = 0; if_getvlantrunk(ifp) != NULL && i < VLAN_N_VID; ++i) { + NET_EPOCH_ENTER(et); ifv = VLAN_DEVAT(ifp, i); + NET_EPOCH_EXIT(et); if (!ifv) continue; irdma_add_ipv4_addr(iwdev, ifv); @@ -361,7 +364,8 @@ irdma_get_route_ifp(struct sockaddr *dst_sin, struct ifnet *netdev, struct nhop_object *nh; if (dst_sin->sa_family == AF_INET6) - nh = fib6_lookup(RT_DEFAULT_FIB, &((struct sockaddr_in6 *)dst_sin)->sin6_addr, 0, NHR_NONE, 0); + nh = fib6_lookup(RT_DEFAULT_FIB, &((struct sockaddr_in6 *)dst_sin)->sin6_addr, + ((struct sockaddr_in6 *)dst_sin)->sin6_scope_id, NHR_NONE, 0); else nh = fib4_lookup(RT_DEFAULT_FIB, ((struct sockaddr_in *)dst_sin)->sin_addr, 0, NHR_NONE, 0); if (!nh || (nh->nh_ifp != netdev && @@ -591,6 +595,188 @@ irdma_sysctl_dcqcn_update(SYSCTL_HANDLER_ARGS) return 0; } +enum irdma_cqp_stats_info { + IRDMA_CQP_REQ_CMDS = 28, + IRDMA_CQP_CMPL_CMDS = 29 +}; + +static int +irdma_sysctl_cqp_stats(SYSCTL_HANDLER_ARGS) +{ + struct irdma_sc_cqp *cqp = (struct irdma_sc_cqp *)arg1; + char rslt[192] = "no cqp available yet"; + int rslt_size = sizeof(rslt) - 1; + int option = (int)arg2; + + if (!cqp) { + return sysctl_handle_string(oidp, rslt, sizeof(rslt), req); + } + + snprintf(rslt, sizeof(rslt), ""); + switch (option) { + case IRDMA_CQP_REQ_CMDS: + snprintf(rslt, rslt_size, "%lu", cqp->requested_ops); + break; + case IRDMA_CQP_CMPL_CMDS: + snprintf(rslt, rslt_size, "%lu", atomic64_read(&cqp->completed_ops)); + break; + } + + return sysctl_handle_string(oidp, rslt, sizeof(rslt), req); +} + +struct irdma_sw_stats_tunable_info { + u8 op_type; + const char name[32]; + const char desc[32]; + uintptr_t value; +}; + +static const struct irdma_sw_stats_tunable_info irdma_sws_list[] = { + {IRDMA_OP_CEQ_DESTROY, "ceq_destroy", "ceq_destroy", 0}, + {IRDMA_OP_AEQ_DESTROY, "aeq_destroy", "aeq_destroy", 0}, + {IRDMA_OP_DELETE_ARP_CACHE_ENTRY, "delete_arp_cache_entry", + "delete_arp_cache_entry", 0}, + {IRDMA_OP_MANAGE_APBVT_ENTRY, "manage_apbvt_entry", + "manage_apbvt_entry", 0}, + {IRDMA_OP_CEQ_CREATE, "ceq_create", "ceq_create", 0}, + {IRDMA_OP_AEQ_CREATE, "aeq_create", "aeq_create", 0}, + {IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY, "manage_qhash_table_entry", + "manage_qhash_table_entry", 0}, + {IRDMA_OP_QP_MODIFY, "qp_modify", "qp_modify", 0}, + {IRDMA_OP_QP_UPLOAD_CONTEXT, "qp_upload_context", "qp_upload_context", + 0}, + {IRDMA_OP_CQ_CREATE, "cq_create", "cq_create", 0}, + {IRDMA_OP_CQ_DESTROY, "cq_destroy", "cq_destroy", 0}, + {IRDMA_OP_QP_CREATE, "qp_create", "qp_create", 0}, + {IRDMA_OP_QP_DESTROY, "qp_destroy", "qp_destroy", 0}, + {IRDMA_OP_ALLOC_STAG, "alloc_stag", "alloc_stag", 0}, + {IRDMA_OP_MR_REG_NON_SHARED, "mr_reg_non_shared", "mr_reg_non_shared", + 0}, + {IRDMA_OP_DEALLOC_STAG, "dealloc_stag", "dealloc_stag", 0}, + {IRDMA_OP_MW_ALLOC, "mw_alloc", "mw_alloc", 0}, + {IRDMA_OP_QP_FLUSH_WQES, "qp_flush_wqes", "qp_flush_wqes", 0}, + {IRDMA_OP_ADD_ARP_CACHE_ENTRY, "add_arp_cache_entry", + "add_arp_cache_entry", 0}, + {IRDMA_OP_MANAGE_PUSH_PAGE, "manage_push_page", "manage_push_page", 0}, + {IRDMA_OP_UPDATE_PE_SDS, "update_pe_sds", "update_pe_sds", 0}, + {IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE, "manage_hmc_pm_func_table", + "manage_hmc_pm_func_table", 0}, + {IRDMA_OP_SUSPEND, "suspend", "suspend", 0}, + {IRDMA_OP_RESUME, "resume", "resume", 0}, + {IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP, "manage_vchnl_req_pble_bp", + "manage_vchnl_req_pble_bp", 0}, + {IRDMA_OP_QUERY_FPM_VAL, "query_fpm_val", "query_fpm_val", 0}, + {IRDMA_OP_COMMIT_FPM_VAL, "commit_fpm_val", "commit_fpm_val", 0}, + {IRDMA_OP_AH_CREATE, "ah_create", "ah_create", 0}, + {IRDMA_OP_AH_MODIFY, "ah_modify", "ah_modify", 0}, + {IRDMA_OP_AH_DESTROY, "ah_destroy", "ah_destroy", 0}, + {IRDMA_OP_MC_CREATE, "mc_create", "mc_create", 0}, + {IRDMA_OP_MC_DESTROY, "mc_destroy", "mc_destroy", 0}, + {IRDMA_OP_MC_MODIFY, "mc_modify", "mc_modify", 0}, + {IRDMA_OP_STATS_ALLOCATE, "stats_allocate", "stats_allocate", 0}, + {IRDMA_OP_STATS_FREE, "stats_free", "stats_free", 0}, + {IRDMA_OP_STATS_GATHER, "stats_gather", "stats_gather", 0}, + {IRDMA_OP_WS_ADD_NODE, "ws_add_node", "ws_add_node", 0}, + {IRDMA_OP_WS_MODIFY_NODE, "ws_modify_node", "ws_modify_node", 0}, + {IRDMA_OP_WS_DELETE_NODE, "ws_delete_node", "ws_delete_node", 0}, + {IRDMA_OP_WS_FAILOVER_START, "ws_failover_start", "ws_failover_start", + 0}, + {IRDMA_OP_WS_FAILOVER_COMPLETE, "ws_failover_complete", + "ws_failover_complete", 0}, + {IRDMA_OP_SET_UP_MAP, "set_up_map", "set_up_map", 0}, + {IRDMA_OP_GEN_AE, "gen_ae", "gen_ae", 0}, + {IRDMA_OP_QUERY_RDMA_FEATURES, "query_rdma_features", + "query_rdma_features", 0}, + {IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY, "alloc_local_mac_entry", + "alloc_local_mac_entry", 0}, + {IRDMA_OP_ADD_LOCAL_MAC_ENTRY, "add_local_mac_entry", + "add_local_mac_entry", 0}, + {IRDMA_OP_DELETE_LOCAL_MAC_ENTRY, "delete_local_mac_entry", + "delete_local_mac_entry", 0}, + {IRDMA_OP_CQ_MODIFY, "cq_modify", "cq_modify", 0} +}; + +static const struct irdma_sw_stats_tunable_info irdma_cmcs_list[] = { + {0, "cm_nodes_created", "cm_nodes_created", + offsetof(struct irdma_cm_core, stats_nodes_created)}, + {0, "cm_nodes_destroyed", "cm_nodes_destroyed", + offsetof(struct irdma_cm_core, stats_nodes_destroyed)}, + {0, "cm_listen_created", "cm_listen_created", + offsetof(struct irdma_cm_core, stats_listen_created)}, + {0, "cm_listen_destroyed", "cm_listen_destroyed", + offsetof(struct irdma_cm_core, stats_listen_destroyed)}, + {0, "cm_listen_nodes_created", "cm_listen_nodes_created", + offsetof(struct irdma_cm_core, stats_listen_nodes_created)}, + {0, "cm_listen_nodes_destroyed", "cm_listen_nodes_destroyed", + offsetof(struct irdma_cm_core, stats_listen_nodes_destroyed)}, + {0, "cm_lpbs", "cm_lpbs", offsetof(struct irdma_cm_core, stats_lpbs)}, + {0, "cm_accepts", "cm_accepts", offsetof(struct irdma_cm_core, + stats_accepts)}, + {0, "cm_rejects", "cm_rejects", offsetof(struct irdma_cm_core, + stats_rejects)}, *** 4485 LINES SKIPPED ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202308181830.37IIUio7046074>