Date: Thu, 22 Dec 2022 01:14:21 GMT From: Eric Joyner <erj@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 777e472cd86b - main - irdma(4): Upgrade driver to 1.1.5-k Message-ID: <202212220114.2BM1ELBg040513@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by erj: URL: https://cgit.FreeBSD.org/src/commit/?id=777e472cd86b9394d07bf96c19dbafc2e1ff4fdc commit 777e472cd86b9394d07bf96c19dbafc2e1ff4fdc Author: Bartosz Sobczak <bartosz.sobczak@intel.com> AuthorDate: 2022-12-22 01:10:15 +0000 Commit: Eric Joyner <erj@FreeBSD.org> CommitDate: 2022-12-22 01:13:42 +0000 irdma(4): Upgrade driver to 1.1.5-k This is to upgrade current irdma driver version (in support of RDMA on Intel(R) Ethernet Controller E810) to 1.1.5-k change summary: - refactor defines for hardware registers - rereg_mr verb added in libirdma - fix print warning during compilation - rt_ros2priority macro fix - irdma.4 validated with mandoc - fixing nd6_resolve usage - added libirdma_query_device - sysctl for irdma version - aeq_alloc_db fix - dwork_flush protected with qp refcount - PFC fixes Signed-off-by: Eric Joyner <erj@FreeBSD.org> Reviewed by: erj@ Sponsored by: Intel Corporation MFC after: 2 weeks Differential Revision: https://reviews.freebsd.org/D36944 --- contrib/ofed/libirdma/abi.h | 10 +- contrib/ofed/libirdma/i40iw_hw.h | 4 +- contrib/ofed/libirdma/irdma.h | 18 +- contrib/ofed/libirdma/irdma_defs.h | 259 ++---- contrib/ofed/libirdma/irdma_uk.c | 774 +++++++++------- contrib/ofed/libirdma/irdma_umain.c | 28 +- contrib/ofed/libirdma/irdma_umain.h | 30 +- contrib/ofed/libirdma/irdma_uquery.h | 50 + contrib/ofed/libirdma/irdma_user.h | 227 ++++- contrib/ofed/libirdma/irdma_uverbs.c | 377 ++++---- contrib/ofed/libirdma/libirdma.map | 8 +- contrib/ofed/libirdma/osdep.h | 17 +- share/man/man4/irdma.4 | 168 ++-- sys/dev/irdma/fbsd_kcompat.c | 17 +- sys/dev/irdma/fbsd_kcompat.h | 19 +- sys/dev/irdma/icrdma.c | 7 +- sys/dev/irdma/icrdma_hw.c | 39 +- sys/dev/irdma/icrdma_hw.h | 35 +- sys/dev/irdma/irdma-abi.h | 2 +- sys/dev/irdma/irdma.h | 158 ++-- sys/dev/irdma/irdma_cm.c | 110 +-- sys/dev/irdma/irdma_cm.h | 22 +- sys/dev/irdma/irdma_ctrl.c | 1330 +++++++++++++-------------- sys/dev/irdma/irdma_defs.h | 1678 ++++++++++------------------------ sys/dev/irdma/irdma_hmc.c | 33 +- sys/dev/irdma/irdma_hmc.h | 12 +- sys/dev/irdma/irdma_hw.c | 205 ++--- sys/dev/irdma/irdma_kcompat.c | 348 ++++--- sys/dev/irdma/irdma_main.h | 23 +- sys/dev/irdma/irdma_pble.c | 4 +- sys/dev/irdma/irdma_protos.h | 13 +- sys/dev/irdma/irdma_puda.c | 146 +-- sys/dev/irdma/irdma_type.h | 99 +- sys/dev/irdma/irdma_uda.c | 79 +- sys/dev/irdma/irdma_uda_d.h | 361 ++------ sys/dev/irdma/irdma_uk.c | 766 +++++++++------- sys/dev/irdma/irdma_user.h | 227 ++++- sys/dev/irdma/irdma_utils.c | 160 +++- sys/dev/irdma/irdma_verbs.c | 706 +++++++++----- sys/dev/irdma/irdma_verbs.h | 27 +- sys/dev/irdma/irdma_ws.c | 4 +- sys/dev/irdma/osdep.h | 6 +- 42 files changed, 4321 insertions(+), 4285 deletions(-) diff --git a/contrib/ofed/libirdma/abi.h b/contrib/ofed/libirdma/abi.h index ff7a2828efe0..e45a7b49caf8 100644 --- a/contrib/ofed/libirdma/abi.h +++ b/contrib/ofed/libirdma/abi.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (C) 2019 - 2020 Intel Corporation + * Copyright (C) 2019 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -145,6 +145,14 @@ struct irdma_ureg_mr { __u16 rq_pages; __u16 sq_pages; +}; +struct irdma_urereg_mr { + struct ibv_rereg_mr ibv_cmd; + __u16 reg_type; /* enum irdma_memreg_type */ + __u16 cq_pages; + __u16 rq_pages; + __u16 sq_pages; + }; struct irdma_ucreate_ah_resp { struct ibv_create_ah_resp ibv_resp; diff --git a/contrib/ofed/libirdma/i40iw_hw.h b/contrib/ofed/libirdma/i40iw_hw.h index 38c7e37c35c9..c51d89a0fcb2 100644 --- a/contrib/ofed/libirdma/i40iw_hw.h +++ b/contrib/ofed/libirdma/i40iw_hw.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2020 Intel Corporation + * Copyright (c) 2015 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -50,11 +50,11 @@ enum i40iw_device_caps_const { I40IW_MAX_CQ_SIZE = 1048575, I40IW_MAX_OUTBOUND_MSG_SIZE = 2147483647, I40IW_MAX_INBOUND_MSG_SIZE = 2147483647, + I40IW_MIN_WQ_SIZE = 4 /* WQEs */, }; #define I40IW_QP_WQE_MIN_SIZE 32 #define I40IW_QP_WQE_MAX_SIZE 128 -#define I40IW_QP_SW_MIN_WQSIZE 4 #define I40IW_MAX_RQ_WQE_SHIFT 2 #define I40IW_MAX_QUANTA_PER_WR 2 diff --git a/contrib/ofed/libirdma/irdma.h b/contrib/ofed/libirdma/irdma.h index 27fa3d53d3e8..1dd09c36c7ea 100644 --- a/contrib/ofed/libirdma/irdma.h +++ b/contrib/ofed/libirdma/irdma.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2017 - 2021 Intel Corporation + * Copyright (c) 2017 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -39,13 +39,15 @@ #define RDMA_BIT2(type, a) ((u##type) 1UL << a) #define RDMA_MASK3(type, mask, shift) ((u##type) mask << shift) #define MAKEMASK(m, s) ((m) << (s)) -#define IRDMA_WQEALLOC_WQE_DESC_INDEX_S 20 -#define IRDMA_WQEALLOC_WQE_DESC_INDEX_M (0xfff << IRDMA_WQEALLOC_WQE_DESC_INDEX_S) + +#define IRDMA_WQEALLOC_WQE_DESC_INDEX_S 20 +#define IRDMA_WQEALLOC_WQE_DESC_INDEX GENMASK(31, 20) enum irdma_vers { - IRDMA_GEN_RSVD, - IRDMA_GEN_1, - IRDMA_GEN_2, + IRDMA_GEN_RSVD = 0, + IRDMA_GEN_1 = 1, + IRDMA_GEN_2 = 2, + IRDMA_GEN_MAX = 2, }; struct irdma_uk_attrs { @@ -58,8 +60,7 @@ struct irdma_uk_attrs { u32 min_hw_cq_size; u32 max_hw_cq_size; u16 max_hw_sq_chunk; - u16 max_hw_wq_size; - u16 min_sw_wq_size; + u16 min_hw_wq_size; u8 hw_rev; }; @@ -68,6 +69,7 @@ struct irdma_hw_attrs { u64 max_hw_outbound_msg_size; u64 max_hw_inbound_msg_size; u64 max_mr_size; + u64 page_size_cap; u32 min_hw_qp_id; u32 min_hw_aeq_size; u32 max_hw_aeq_size; diff --git a/contrib/ofed/libirdma/irdma_defs.h b/contrib/ofed/libirdma/irdma_defs.h index 8fb9f1e2b622..932993fd44ce 100644 --- a/contrib/ofed/libirdma/irdma_defs.h +++ b/contrib/ofed/libirdma/irdma_defs.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2021 Intel Corporation + * Copyright (c) 2015 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -75,7 +75,6 @@ #define IRDMA_CQE_QTYPE_RQ 0 #define IRDMA_CQE_QTYPE_SQ 1 -#define IRDMA_QP_SW_MIN_WQSIZE 8u /* in WRs*/ #define IRDMA_QP_WQE_MIN_SIZE 32 #define IRDMA_QP_WQE_MAX_SIZE 256 #define IRDMA_QP_WQE_MIN_QUANTA 1 @@ -85,9 +84,11 @@ #define IRDMA_SQ_RSVD 258 #define IRDMA_RQ_RSVD 1 -#define IRDMA_FEATURE_RTS_AE 1ULL -#define IRDMA_FEATURE_CQ_RESIZE 2ULL -#define IRDMA_FEATURE_RELAX_RQ_ORDER 4ULL +#define IRDMA_FEATURE_RTS_AE BIT_ULL(0) +#define IRDMA_FEATURE_CQ_RESIZE BIT_ULL(1) +#define IRDMA_FEATURE_RELAX_RQ_ORDER BIT_ULL(2) +#define IRDMA_FEATURE_64_BYTE_CQE BIT_ULL(5) + #define IRDMAQP_OP_RDMA_WRITE 0x00 #define IRDMAQP_OP_RDMA_READ 0x01 #define IRDMAQP_OP_RDMA_SEND 0x03 @@ -106,262 +107,198 @@ #define LS_32_1(val, bits) ((u32)((val) << (bits))) #define RS_32_1(val, bits) ((u32)((val) >> (bits))) #endif -#define LS_64(val, field) (((u64)(val) << field ## _S) & (field ## _M)) -#define RS_64(val, field) ((u64)((val) & field ## _M) >> field ## _S) -#define LS_32(val, field) (((val) << field ## _S) & (field ## _M)) -#define RS_32(val, field) (((val) & field ## _M) >> field ## _S) +#ifndef GENMASK_ULL +#define GENMASK_ULL(high, low) ((0xFFFFFFFFFFFFFFFFULL >> (64ULL - ((high) - (low) + 1ULL))) << (low)) +#endif /* GENMASK_ULL */ +#ifndef GENMASK +#define GENMASK(high, low) ((0xFFFFFFFFUL >> (32UL - ((high) - (low) + 1UL))) << (low)) +#endif /* GENMASK */ +#ifndef FIELD_PREP +#define FIELD_PREP(mask, val) (((u64)(val) << mask##_S) & (mask)) +#define FIELD_GET(mask, val) (((val) & mask) >> mask##_S) +#endif /* FIELD_PREP */ #define IRDMA_CQPHC_QPCTX_S 0 -#define IRDMA_CQPHC_QPCTX_M \ - (0xffffffffffffffffULL << IRDMA_CQPHC_QPCTX_S) - -/* iWARP QP Doorbell shadow area */ +#define IRDMA_CQPHC_QPCTX GENMASK_ULL(63, 0) #define IRDMA_QP_DBSA_HW_SQ_TAIL_S 0 -#define IRDMA_QP_DBSA_HW_SQ_TAIL_M \ - (0x7fffULL << IRDMA_QP_DBSA_HW_SQ_TAIL_S) - -/* Completion Queue Doorbell shadow area */ +#define IRDMA_QP_DBSA_HW_SQ_TAIL GENMASK_ULL(14, 0) #define IRDMA_CQ_DBSA_CQEIDX_S 0 -#define IRDMA_CQ_DBSA_CQEIDX_M (0xfffffULL << IRDMA_CQ_DBSA_CQEIDX_S) - +#define IRDMA_CQ_DBSA_CQEIDX GENMASK_ULL(19, 0) #define IRDMA_CQ_DBSA_SW_CQ_SELECT_S 0 -#define IRDMA_CQ_DBSA_SW_CQ_SELECT_M \ - (0x3fffULL << IRDMA_CQ_DBSA_SW_CQ_SELECT_S) - +#define IRDMA_CQ_DBSA_SW_CQ_SELECT GENMASK_ULL(13, 0) #define IRDMA_CQ_DBSA_ARM_NEXT_S 14 -#define IRDMA_CQ_DBSA_ARM_NEXT_M BIT_ULL(IRDMA_CQ_DBSA_ARM_NEXT_S) - +#define IRDMA_CQ_DBSA_ARM_NEXT BIT_ULL(14) #define IRDMA_CQ_DBSA_ARM_NEXT_SE_S 15 -#define IRDMA_CQ_DBSA_ARM_NEXT_SE_M BIT_ULL(IRDMA_CQ_DBSA_ARM_NEXT_SE_S) - +#define IRDMA_CQ_DBSA_ARM_NEXT_SE BIT_ULL(15) #define IRDMA_CQ_DBSA_ARM_SEQ_NUM_S 16 -#define IRDMA_CQ_DBSA_ARM_SEQ_NUM_M \ - (0x3ULL << IRDMA_CQ_DBSA_ARM_SEQ_NUM_S) +#define IRDMA_CQ_DBSA_ARM_SEQ_NUM GENMASK_ULL(17, 16) /* CQP and iWARP Completion Queue */ #define IRDMA_CQ_QPCTX_S IRDMA_CQPHC_QPCTX_S -#define IRDMA_CQ_QPCTX_M IRDMA_CQPHC_QPCTX_M +#define IRDMA_CQ_QPCTX IRDMA_CQPHC_QPCTX #define IRDMA_CQ_MINERR_S 0 -#define IRDMA_CQ_MINERR_M (0xffffULL << IRDMA_CQ_MINERR_S) - +#define IRDMA_CQ_MINERR GENMASK_ULL(15, 0) #define IRDMA_CQ_MAJERR_S 16 -#define IRDMA_CQ_MAJERR_M (0xffffULL << IRDMA_CQ_MAJERR_S) - +#define IRDMA_CQ_MAJERR GENMASK_ULL(31, 16) #define IRDMA_CQ_WQEIDX_S 32 -#define IRDMA_CQ_WQEIDX_M (0x7fffULL << IRDMA_CQ_WQEIDX_S) - +#define IRDMA_CQ_WQEIDX GENMASK_ULL(46, 32) #define IRDMA_CQ_EXTCQE_S 50 -#define IRDMA_CQ_EXTCQE_M BIT_ULL(IRDMA_CQ_EXTCQE_S) - +#define IRDMA_CQ_EXTCQE BIT_ULL(50) #define IRDMA_OOO_CMPL_S 54 -#define IRDMA_OOO_CMPL_M BIT_ULL(IRDMA_OOO_CMPL_S) - +#define IRDMA_OOO_CMPL BIT_ULL(54) #define IRDMA_CQ_ERROR_S 55 -#define IRDMA_CQ_ERROR_M BIT_ULL(IRDMA_CQ_ERROR_S) - +#define IRDMA_CQ_ERROR BIT_ULL(55) #define IRDMA_CQ_SQ_S 62 -#define IRDMA_CQ_SQ_M BIT_ULL(IRDMA_CQ_SQ_S) +#define IRDMA_CQ_SQ BIT_ULL(62) #define IRDMA_CQ_VALID_S 63 -#define IRDMA_CQ_VALID_M BIT_ULL(IRDMA_CQ_VALID_S) - -#define IRDMA_CQ_IMMVALID_S 62 -#define IRDMA_CQ_IMMVALID_M BIT_ULL(IRDMA_CQ_IMMVALID_S) - +#define IRDMA_CQ_VALID BIT_ULL(63) +#define IRDMA_CQ_IMMVALID BIT_ULL(62) #define IRDMA_CQ_UDSMACVALID_S 61 -#define IRDMA_CQ_UDSMACVALID_M BIT_ULL(IRDMA_CQ_UDSMACVALID_S) - +#define IRDMA_CQ_UDSMACVALID BIT_ULL(61) #define IRDMA_CQ_UDVLANVALID_S 60 -#define IRDMA_CQ_UDVLANVALID_M BIT_ULL(IRDMA_CQ_UDVLANVALID_S) - +#define IRDMA_CQ_UDVLANVALID BIT_ULL(60) #define IRDMA_CQ_UDSMAC_S 0 -#define IRDMA_CQ_UDSMAC_M (0xffffffffffffULL << IRDMA_CQ_UDSMAC_S) - +#define IRDMA_CQ_UDSMAC GENMASK_ULL(47, 0) #define IRDMA_CQ_UDVLAN_S 48 -#define IRDMA_CQ_UDVLAN_M (0xffffULL << IRDMA_CQ_UDVLAN_S) +#define IRDMA_CQ_UDVLAN GENMASK_ULL(63, 48) #define IRDMA_CQ_IMMDATA_S 0 -#define IRDMA_CQ_IMMDATA_M (0xffffffffffffffffULL << IRDMA_CQ_IMMVALID_S) - +#define IRDMA_CQ_IMMVALID_S 62 +#define IRDMA_CQ_IMMDATA GENMASK_ULL(125, 62) #define IRDMA_CQ_IMMDATALOW32_S 0 -#define IRDMA_CQ_IMMDATALOW32_M (0xffffffffULL << IRDMA_CQ_IMMDATALOW32_S) - +#define IRDMA_CQ_IMMDATALOW32 GENMASK_ULL(31, 0) #define IRDMA_CQ_IMMDATAUP32_S 32 -#define IRDMA_CQ_IMMDATAUP32_M (0xffffffffULL << IRDMA_CQ_IMMDATAUP32_S) - +#define IRDMA_CQ_IMMDATAUP32 GENMASK_ULL(63, 32) #define IRDMACQ_PAYLDLEN_S 0 -#define IRDMACQ_PAYLDLEN_M (0xffffffffULL << IRDMACQ_PAYLDLEN_S) - -#define IRDMACQ_TCPSEQNUMRTT_S 32 -#define IRDMACQ_TCPSEQNUMRTT_M (0xffffffffULL << IRDMACQ_TCPSEQNUMRTT_S) - +#define IRDMACQ_PAYLDLEN GENMASK_ULL(31, 0) +#define IRDMACQ_TCPSQN_ROCEPSN_RTT_TS_S 32 +#define IRDMACQ_TCPSQN_ROCEPSN_RTT_TS GENMASK_ULL(63, 32) #define IRDMACQ_INVSTAG_S 0 -#define IRDMACQ_INVSTAG_M (0xffffffffULL << IRDMACQ_INVSTAG_S) - +#define IRDMACQ_INVSTAG GENMASK_ULL(31, 0) #define IRDMACQ_QPID_S 32 -#define IRDMACQ_QPID_M (0xffffffULL << IRDMACQ_QPID_S) +#define IRDMACQ_QPID GENMASK_ULL(55, 32) #define IRDMACQ_UDSRCQPN_S 0 -#define IRDMACQ_UDSRCQPN_M (0xffffffffULL << IRDMACQ_UDSRCQPN_S) - +#define IRDMACQ_UDSRCQPN GENMASK_ULL(31, 0) #define IRDMACQ_PSHDROP_S 51 -#define IRDMACQ_PSHDROP_M BIT_ULL(IRDMACQ_PSHDROP_S) - +#define IRDMACQ_PSHDROP BIT_ULL(51) #define IRDMACQ_STAG_S 53 -#define IRDMACQ_STAG_M BIT_ULL(IRDMACQ_STAG_S) - +#define IRDMACQ_STAG BIT_ULL(53) #define IRDMACQ_IPV4_S 53 -#define IRDMACQ_IPV4_M BIT_ULL(IRDMACQ_IPV4_S) - +#define IRDMACQ_IPV4 BIT_ULL(53) #define IRDMACQ_SOEVENT_S 54 -#define IRDMACQ_SOEVENT_M BIT_ULL(IRDMACQ_SOEVENT_S) - +#define IRDMACQ_SOEVENT BIT_ULL(54) #define IRDMACQ_OP_S 56 -#define IRDMACQ_OP_M (0x3fULL << IRDMACQ_OP_S) +#define IRDMACQ_OP GENMASK_ULL(61, 56) /* Manage Push Page - MPP */ #define IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1 0xffff #define IRDMA_INVALID_PUSH_PAGE_INDEX 0xffffffff -/* iwarp QP SQ WQE common fields */ #define IRDMAQPSQ_OPCODE_S 32 -#define IRDMAQPSQ_OPCODE_M (0x3fULL << IRDMAQPSQ_OPCODE_S) - +#define IRDMAQPSQ_OPCODE GENMASK_ULL(37, 32) #define IRDMAQPSQ_COPY_HOST_PBL_S 43 -#define IRDMAQPSQ_COPY_HOST_PBL_M BIT_ULL(IRDMAQPSQ_COPY_HOST_PBL_S) - +#define IRDMAQPSQ_COPY_HOST_PBL BIT_ULL(43) #define IRDMAQPSQ_ADDFRAGCNT_S 38 -#define IRDMAQPSQ_ADDFRAGCNT_M (0xfULL << IRDMAQPSQ_ADDFRAGCNT_S) - +#define IRDMAQPSQ_ADDFRAGCNT GENMASK_ULL(41, 38) #define IRDMAQPSQ_PUSHWQE_S 56 -#define IRDMAQPSQ_PUSHWQE_M BIT_ULL(IRDMAQPSQ_PUSHWQE_S) - +#define IRDMAQPSQ_PUSHWQE BIT_ULL(56) #define IRDMAQPSQ_STREAMMODE_S 58 -#define IRDMAQPSQ_STREAMMODE_M BIT_ULL(IRDMAQPSQ_STREAMMODE_S) - +#define IRDMAQPSQ_STREAMMODE BIT_ULL(58) #define IRDMAQPSQ_WAITFORRCVPDU_S 59 -#define IRDMAQPSQ_WAITFORRCVPDU_M BIT_ULL(IRDMAQPSQ_WAITFORRCVPDU_S) - +#define IRDMAQPSQ_WAITFORRCVPDU BIT_ULL(59) #define IRDMAQPSQ_READFENCE_S 60 -#define IRDMAQPSQ_READFENCE_M BIT_ULL(IRDMAQPSQ_READFENCE_S) - +#define IRDMAQPSQ_READFENCE BIT_ULL(60) #define IRDMAQPSQ_LOCALFENCE_S 61 -#define IRDMAQPSQ_LOCALFENCE_M BIT_ULL(IRDMAQPSQ_LOCALFENCE_S) - +#define IRDMAQPSQ_LOCALFENCE BIT_ULL(61) #define IRDMAQPSQ_UDPHEADER_S 61 -#define IRDMAQPSQ_UDPHEADER_M BIT_ULL(IRDMAQPSQ_UDPHEADER_S) - +#define IRDMAQPSQ_UDPHEADER BIT_ULL(61) #define IRDMAQPSQ_L4LEN_S 42 -#define IRDMAQPSQ_L4LEN_M ((u64)0xF << IRDMAQPSQ_L4LEN_S) - +#define IRDMAQPSQ_L4LEN GENMASK_ULL(45, 42) #define IRDMAQPSQ_SIGCOMPL_S 62 -#define IRDMAQPSQ_SIGCOMPL_M BIT_ULL(IRDMAQPSQ_SIGCOMPL_S) - +#define IRDMAQPSQ_SIGCOMPL BIT_ULL(62) #define IRDMAQPSQ_VALID_S 63 -#define IRDMAQPSQ_VALID_M BIT_ULL(IRDMAQPSQ_VALID_S) +#define IRDMAQPSQ_VALID BIT_ULL(63) #define IRDMAQPSQ_FRAG_TO_S IRDMA_CQPHC_QPCTX_S -#define IRDMAQPSQ_FRAG_TO_M IRDMA_CQPHC_QPCTX_M - +#define IRDMAQPSQ_FRAG_TO IRDMA_CQPHC_QPCTX #define IRDMAQPSQ_FRAG_VALID_S 63 -#define IRDMAQPSQ_FRAG_VALID_M BIT_ULL(IRDMAQPSQ_FRAG_VALID_S) - +#define IRDMAQPSQ_FRAG_VALID BIT_ULL(63) #define IRDMAQPSQ_FRAG_LEN_S 32 -#define IRDMAQPSQ_FRAG_LEN_M (0x7fffffffULL << IRDMAQPSQ_FRAG_LEN_S) - +#define IRDMAQPSQ_FRAG_LEN GENMASK_ULL(62, 32) #define IRDMAQPSQ_FRAG_STAG_S 0 -#define IRDMAQPSQ_FRAG_STAG_M (0xffffffffULL << IRDMAQPSQ_FRAG_STAG_S) - +#define IRDMAQPSQ_FRAG_STAG GENMASK_ULL(31, 0) #define IRDMAQPSQ_GEN1_FRAG_LEN_S 0 -#define IRDMAQPSQ_GEN1_FRAG_LEN_M (0xffffffffULL << IRDMAQPSQ_GEN1_FRAG_LEN_S) - +#define IRDMAQPSQ_GEN1_FRAG_LEN GENMASK_ULL(31, 0) #define IRDMAQPSQ_GEN1_FRAG_STAG_S 32 -#define IRDMAQPSQ_GEN1_FRAG_STAG_M (0xffffffffULL << IRDMAQPSQ_GEN1_FRAG_STAG_S) - +#define IRDMAQPSQ_GEN1_FRAG_STAG GENMASK_ULL(63, 32) #define IRDMAQPSQ_REMSTAGINV_S 0 -#define IRDMAQPSQ_REMSTAGINV_M (0xffffffffULL << IRDMAQPSQ_REMSTAGINV_S) - +#define IRDMAQPSQ_REMSTAGINV GENMASK_ULL(31, 0) #define IRDMAQPSQ_DESTQKEY_S 0 -#define IRDMAQPSQ_DESTQKEY_M (0xffffffffULL << IRDMAQPSQ_DESTQKEY_S) - +#define IRDMAQPSQ_DESTQKEY GENMASK_ULL(31, 0) #define IRDMAQPSQ_DESTQPN_S 32 -#define IRDMAQPSQ_DESTQPN_M (0x00ffffffULL << IRDMAQPSQ_DESTQPN_S) - +#define IRDMAQPSQ_DESTQPN GENMASK_ULL(55, 32) #define IRDMAQPSQ_AHID_S 0 -#define IRDMAQPSQ_AHID_M (0x0001ffffULL << IRDMAQPSQ_AHID_S) - +#define IRDMAQPSQ_AHID GENMASK_ULL(16, 0) #define IRDMAQPSQ_INLINEDATAFLAG_S 57 -#define IRDMAQPSQ_INLINEDATAFLAG_M BIT_ULL(IRDMAQPSQ_INLINEDATAFLAG_S) +#define IRDMAQPSQ_INLINEDATAFLAG BIT_ULL(57) #define IRDMA_INLINE_VALID_S 7 - #define IRDMAQPSQ_INLINEDATALEN_S 48 -#define IRDMAQPSQ_INLINEDATALEN_M \ - (0xffULL << IRDMAQPSQ_INLINEDATALEN_S) +#define IRDMAQPSQ_INLINEDATALEN GENMASK_ULL(55, 48) #define IRDMAQPSQ_IMMDATAFLAG_S 47 -#define IRDMAQPSQ_IMMDATAFLAG_M \ - BIT_ULL(IRDMAQPSQ_IMMDATAFLAG_S) +#define IRDMAQPSQ_IMMDATAFLAG BIT_ULL(47) #define IRDMAQPSQ_REPORTRTT_S 46 -#define IRDMAQPSQ_REPORTRTT_M \ - BIT_ULL(IRDMAQPSQ_REPORTRTT_S) +#define IRDMAQPSQ_REPORTRTT BIT_ULL(46) #define IRDMAQPSQ_IMMDATA_S 0 -#define IRDMAQPSQ_IMMDATA_M \ - (0xffffffffffffffffULL << IRDMAQPSQ_IMMDATA_S) - -/* rdma write */ +#define IRDMAQPSQ_IMMDATA GENMASK_ULL(63, 0) #define IRDMAQPSQ_REMSTAG_S 0 -#define IRDMAQPSQ_REMSTAG_M (0xffffffffULL << IRDMAQPSQ_REMSTAG_S) +#define IRDMAQPSQ_REMSTAG GENMASK_ULL(31, 0) #define IRDMAQPSQ_REMTO_S IRDMA_CQPHC_QPCTX_S -#define IRDMAQPSQ_REMTO_M IRDMA_CQPHC_QPCTX_M +#define IRDMAQPSQ_REMTO IRDMA_CQPHC_QPCTX -/* memory window */ #define IRDMAQPSQ_STAGRIGHTS_S 48 -#define IRDMAQPSQ_STAGRIGHTS_M (0x1fULL << IRDMAQPSQ_STAGRIGHTS_S) - +#define IRDMAQPSQ_STAGRIGHTS GENMASK_ULL(52, 48) #define IRDMAQPSQ_VABASEDTO_S 53 -#define IRDMAQPSQ_VABASEDTO_M BIT_ULL(IRDMAQPSQ_VABASEDTO_S) - +#define IRDMAQPSQ_VABASEDTO BIT_ULL(53) #define IRDMAQPSQ_MEMWINDOWTYPE_S 54 -#define IRDMAQPSQ_MEMWINDOWTYPE_M BIT_ULL(IRDMAQPSQ_MEMWINDOWTYPE_S) +#define IRDMAQPSQ_MEMWINDOWTYPE BIT_ULL(54) #define IRDMAQPSQ_MWLEN_S IRDMA_CQPHC_QPCTX_S -#define IRDMAQPSQ_MWLEN_M IRDMA_CQPHC_QPCTX_M - +#define IRDMAQPSQ_MWLEN IRDMA_CQPHC_QPCTX #define IRDMAQPSQ_PARENTMRSTAG_S 32 -#define IRDMAQPSQ_PARENTMRSTAG_M \ - (0xffffffffULL << IRDMAQPSQ_PARENTMRSTAG_S) - +#define IRDMAQPSQ_PARENTMRSTAG GENMASK_ULL(63, 32) #define IRDMAQPSQ_MWSTAG_S 0 -#define IRDMAQPSQ_MWSTAG_M (0xffffffffULL << IRDMAQPSQ_MWSTAG_S) +#define IRDMAQPSQ_MWSTAG GENMASK_ULL(31, 0) #define IRDMAQPSQ_BASEVA_TO_FBO_S IRDMA_CQPHC_QPCTX_S -#define IRDMAQPSQ_BASEVA_TO_FBO_M IRDMA_CQPHC_QPCTX_M +#define IRDMAQPSQ_BASEVA_TO_FBO IRDMA_CQPHC_QPCTX -/* Local Invalidate */ #define IRDMAQPSQ_LOCSTAG_S 0 -#define IRDMAQPSQ_LOCSTAG_M (0xffffffffULL << IRDMAQPSQ_LOCSTAG_S) +#define IRDMAQPSQ_LOCSTAG GENMASK_ULL(31, 0) /* iwarp QP RQ WQE common fields */ #define IRDMAQPRQ_ADDFRAGCNT_S IRDMAQPSQ_ADDFRAGCNT_S -#define IRDMAQPRQ_ADDFRAGCNT_M IRDMAQPSQ_ADDFRAGCNT_M +#define IRDMAQPRQ_ADDFRAGCNT IRDMAQPSQ_ADDFRAGCNT #define IRDMAQPRQ_VALID_S IRDMAQPSQ_VALID_S -#define IRDMAQPRQ_VALID_M IRDMAQPSQ_VALID_M +#define IRDMAQPRQ_VALID IRDMAQPSQ_VALID #define IRDMAQPRQ_COMPLCTX_S IRDMA_CQPHC_QPCTX_S -#define IRDMAQPRQ_COMPLCTX_M IRDMA_CQPHC_QPCTX_M +#define IRDMAQPRQ_COMPLCTX IRDMA_CQPHC_QPCTX #define IRDMAQPRQ_FRAG_LEN_S IRDMAQPSQ_FRAG_LEN_S -#define IRDMAQPRQ_FRAG_LEN_M IRDMAQPSQ_FRAG_LEN_M +#define IRDMAQPRQ_FRAG_LEN IRDMAQPSQ_FRAG_LEN #define IRDMAQPRQ_STAG_S IRDMAQPSQ_FRAG_STAG_S -#define IRDMAQPRQ_STAG_M IRDMAQPSQ_FRAG_STAG_M +#define IRDMAQPRQ_STAG IRDMAQPSQ_FRAG_STAG #define IRDMAQPRQ_TO_S IRDMAQPSQ_FRAG_TO_S -#define IRDMAQPRQ_TO_M IRDMAQPSQ_FRAG_TO_M +#define IRDMAQPRQ_TO IRDMAQPSQ_FRAG_TO #define IRDMAPFINT_OICR_HMC_ERR_M BIT(26) #define IRDMAPFINT_OICR_PE_PUSH_M BIT(27) @@ -500,6 +437,12 @@ IRDMA_RING_MOVE_HEAD(_ring, _retcode); \ } +enum irdma_protocol_used { + IRDMA_ANY_PROTOCOL = 0, + IRDMA_IWARP_PROTOCOL_ONLY = 1, + IRDMA_ROCE_PROTOCOL_ONLY = 2, +}; + enum irdma_qp_wqe_size { IRDMA_WQE_SIZE_32 = 32, IRDMA_WQE_SIZE_64 = 64, diff --git a/contrib/ofed/libirdma/irdma_uk.c b/contrib/ofed/libirdma/irdma_uk.c index 2f77c132d296..5201ad692dc1 100644 --- a/contrib/ofed/libirdma/irdma_uk.c +++ b/contrib/ofed/libirdma/irdma_uk.c @@ -51,15 +51,15 @@ irdma_set_fragment(__le64 * wqe, u32 offset, struct irdma_sge *sge, { if (sge) { set_64bit_val(wqe, offset, - LS_64(sge->tag_off, IRDMAQPSQ_FRAG_TO)); + FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off)); set_64bit_val(wqe, offset + IRDMA_BYTE_8, - LS_64(valid, IRDMAQPSQ_VALID) | - LS_64(sge->len, IRDMAQPSQ_FRAG_LEN) | - LS_64(sge->stag, IRDMAQPSQ_FRAG_STAG)); + FIELD_PREP(IRDMAQPSQ_VALID, valid) | + FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->len) | + FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->stag)); } else { set_64bit_val(wqe, offset, 0); set_64bit_val(wqe, offset + IRDMA_BYTE_8, - LS_64(valid, IRDMAQPSQ_VALID)); + FIELD_PREP(IRDMAQPSQ_VALID, valid)); } } @@ -76,16 +76,26 @@ irdma_set_fragment_gen_1(__le64 * wqe, u32 offset, { if (sge) { set_64bit_val(wqe, offset, - LS_64(sge->tag_off, IRDMAQPSQ_FRAG_TO)); + FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off)); set_64bit_val(wqe, offset + IRDMA_BYTE_8, - LS_64(sge->len, IRDMAQPSQ_GEN1_FRAG_LEN) | - LS_64(sge->stag, IRDMAQPSQ_GEN1_FRAG_STAG)); + FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->len) | + FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->stag)); } else { set_64bit_val(wqe, offset, 0); set_64bit_val(wqe, offset + IRDMA_BYTE_8, 0); } } +/** + * irdma_nop_hdr - Format header section of noop WQE + * @qp: hw qp ptr + */ +static inline u64 irdma_nop_hdr(struct irdma_qp_uk *qp){ + return FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) | + FIELD_PREP(IRDMAQPSQ_SIGCOMPL, false) | + FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); +} + /** * irdma_nop_1 - insert a NOP wqe * @qp: hw qp ptr @@ -93,10 +103,8 @@ irdma_set_fragment_gen_1(__le64 * wqe, u32 offset, static int irdma_nop_1(struct irdma_qp_uk *qp) { - u64 hdr; __le64 *wqe; u32 wqe_idx; - bool signaled = false; if (!qp->sq_ring.head) return EINVAL; @@ -110,14 +118,10 @@ irdma_nop_1(struct irdma_qp_uk *qp) set_64bit_val(wqe, IRDMA_BYTE_8, 0); set_64bit_val(wqe, IRDMA_BYTE_16, 0); - hdr = LS_64(IRDMAQP_OP_NOP, IRDMAQPSQ_OPCODE) | - LS_64(signaled, IRDMAQPSQ_SIGCOMPL) | - LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID); - /* make sure WQE is written before valid bit is set */ udma_to_device_barrier(); - set_64bit_val(wqe, IRDMA_BYTE_24, hdr); + set_64bit_val(wqe, IRDMA_BYTE_24, irdma_nop_hdr(qp)); return 0; } @@ -160,7 +164,7 @@ irdma_uk_qp_post_wr(struct irdma_qp_uk *qp) /* read the doorbell shadow area */ get_64bit_val(qp->shadow_area, IRDMA_BYTE_0, &temp); - hw_sq_tail = (u32)RS_64(temp, IRDMA_QP_DBSA_HW_SQ_TAIL); + hw_sq_tail = (u32)FIELD_GET(IRDMA_QP_DBSA_HW_SQ_TAIL, temp); sw_sq_head = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); if (sw_sq_head != qp->initial_ring.head) { if (qp->push_dropped) { @@ -191,7 +195,7 @@ static void irdma_qp_ring_push_db(struct irdma_qp_uk *qp, u32 wqe_idx) { set_32bit_val(qp->push_db, 0, - LS_32(wqe_idx >> 3, IRDMA_WQEALLOC_WQE_DESC_INDEX) | qp->qp_id); + FIELD_PREP(IRDMA_WQEALLOC_WQE_DESC_INDEX, wqe_idx >> 3) | qp->qp_id); qp->initial_ring.head = qp->sq_ring.head; qp->push_mode = true; qp->push_dropped = false; @@ -220,31 +224,32 @@ irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 * wqe, u16 quanta, * irdma_qp_get_next_send_wqe - pad with NOP if needed, return where next WR should go * @qp: hw qp ptr * @wqe_idx: return wqe index - * @quanta: size of WR in quanta + * @quanta: (in/out) ptr to size of WR in quanta. Modified in case pad is needed * @total_size: size of WR in bytes * @info: info on WR */ __le64 * irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx, - u16 quanta, u32 total_size, + u16 *quanta, u32 total_size, struct irdma_post_sq_info *info) { __le64 *wqe; __le64 *wqe_0 = NULL; u32 nop_wqe_idx; - u16 avail_quanta; + u16 avail_quanta, wqe_quanta = *quanta; u16 i; avail_quanta = qp->uk_attrs->max_hw_sq_chunk - (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) % qp->uk_attrs->max_hw_sq_chunk); - if (quanta <= avail_quanta) { + + if (*quanta <= avail_quanta) { /* WR fits in current chunk */ - if (quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring)) + if (*quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring)) return NULL; } else { /* Need to pad with NOP */ - if (quanta + avail_quanta > + if (*quanta + avail_quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring)) return NULL; @@ -262,17 +267,19 @@ irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx, if (!*wqe_idx) qp->swqe_polarity = !qp->swqe_polarity; - IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, quanta); + IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, *quanta); + + irdma_clr_wqes(qp, *wqe_idx); wqe = qp->sq_base[*wqe_idx].elem; - if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && quanta == 1 && + if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && wqe_quanta == 1 && (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) { wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem; - wqe_0[3] = htole64(LS_64(!qp->swqe_polarity, IRDMAQPSQ_VALID)); + wqe_0[3] = htole64(FIELD_PREP(IRDMAQPSQ_VALID, !qp->swqe_polarity)); } qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id; qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size; - qp->sq_wrtrk_array[*wqe_idx].quanta = quanta; + qp->sq_wrtrk_array[*wqe_idx].quanta = wqe_quanta; qp->sq_wrtrk_array[*wqe_idx].signaled = info->signaled; return wqe; @@ -344,20 +351,17 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, if (ret_code) return ret_code; - wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size, - info); + wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info); if (!wqe) return ENOSPC; - irdma_clr_wqes(qp, wqe_idx); - qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled; set_64bit_val(wqe, IRDMA_BYTE_16, - LS_64(op_info->rem_addr.tag_off, IRDMAQPSQ_FRAG_TO)); + FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off)); if (info->imm_data_valid) { set_64bit_val(wqe, IRDMA_BYTE_0, - LS_64(info->imm_data, IRDMAQPSQ_IMMDATA)); + FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data)); i = 0; } else { qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, @@ -382,28 +386,24 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, ++addl_frag_cnt; } - if (!op_info->rem_addr.stag && !total_size) - op_info->rem_addr.stag = 0x1234; - hdr = LS_64(op_info->rem_addr.stag, IRDMAQPSQ_REMSTAG) | - LS_64(info->op_type, IRDMAQPSQ_OPCODE) | - LS_64((info->imm_data_valid ? 1 : 0), IRDMAQPSQ_IMMDATAFLAG) | - LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) | - LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) | - LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) | - LS_64(read_fence, IRDMAQPSQ_READFENCE) | - LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) | - LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) | - LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID); + hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) | + FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | + FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) | + FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) | + FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) | + FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) | + FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) | + FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | + FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | + FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ set_64bit_val(wqe, IRDMA_BYTE_24, hdr); - if (info->push_wqe) { + if (info->push_wqe) irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); - } else { - if (post_sq) - irdma_uk_qp_post_wr(qp); - } + else if (post_sq) + irdma_uk_qp_post_wr(qp); return 0; } @@ -443,8 +443,7 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, if (ret_code) return ret_code; - wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size, - info); + wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info); if (!wqe) return ENOSPC; @@ -453,8 +452,6 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, qp->ord_cnt = 0; } - irdma_clr_wqes(qp, wqe_idx); - qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled; addl_frag_cnt = op_info->num_lo_sges > 1 ? (op_info->num_lo_sges - 1) : 0; @@ -478,28 +475,26 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, ++addl_frag_cnt; } set_64bit_val(wqe, IRDMA_BYTE_16, - LS_64(op_info->rem_addr.tag_off, IRDMAQPSQ_FRAG_TO)); - hdr = LS_64(op_info->rem_addr.stag, IRDMAQPSQ_REMSTAG) | - LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) | - LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) | - LS_64((inv_stag ? IRDMAQP_OP_RDMA_READ_LOC_INV : IRDMAQP_OP_RDMA_READ), - IRDMAQPSQ_OPCODE) | - LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) | - LS_64(info->read_fence || qp->force_fence || ord_fence ? 1 : 0, - IRDMAQPSQ_READFENCE) | - LS_64(local_fence, IRDMAQPSQ_LOCALFENCE) | - LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) | - LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID); + FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off)); + hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) | + FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) | + FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) | + FIELD_PREP(IRDMAQPSQ_OPCODE, + (inv_stag ? IRDMAQP_OP_RDMA_READ_LOC_INV : IRDMAQP_OP_RDMA_READ)) | + FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) | + FIELD_PREP(IRDMAQPSQ_READFENCE, + info->read_fence || ord_fence ? 1 : 0) | + FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) | + FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | + FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ set_64bit_val(wqe, IRDMA_BYTE_24, hdr); - if (info->push_wqe) { + if (info->push_wqe) irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); - } else { - if (post_sq) - irdma_uk_qp_post_wr(qp); - } + else if (post_sq) + irdma_uk_qp_post_wr(qp); return 0; } @@ -540,21 +535,19 @@ irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, if (ret_code) return ret_code; - wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size, - info); + wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info); if (!wqe) return ENOSPC; - irdma_clr_wqes(qp, wqe_idx); - read_fence |= info->read_fence; addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0; if (info->imm_data_valid) { set_64bit_val(wqe, IRDMA_BYTE_0, - LS_64(info->imm_data, IRDMAQPSQ_IMMDATA)); + FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data)); i = 0; } else { - qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, op_info->sg_list, + qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, + frag_cnt ? op_info->sg_list : NULL, qp->swqe_polarity); i = 1; } @@ -575,31 +568,30 @@ irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, } set_64bit_val(wqe, IRDMA_BYTE_16, - LS_64(op_info->qkey, IRDMAQPSQ_DESTQKEY) | - LS_64(op_info->dest_qp, IRDMAQPSQ_DESTQPN)); - hdr = LS_64(info->stag_to_inv, IRDMAQPSQ_REMSTAG) | - LS_64(op_info->ah_id, IRDMAQPSQ_AHID) | - LS_64((info->imm_data_valid ? 1 : 0), IRDMAQPSQ_IMMDATAFLAG) | - LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) | - LS_64(info->op_type, IRDMAQPSQ_OPCODE) | - LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) | - LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) | - LS_64(read_fence, IRDMAQPSQ_READFENCE) | - LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) | - LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) | - LS_64(info->udp_hdr, IRDMAQPSQ_UDPHEADER) | - LS_64(info->l4len, IRDMAQPSQ_L4LEN) | - LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID); + FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) | + FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp)); + hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) | + FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) | + FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, + (info->imm_data_valid ? 1 : 0)) | + FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) | + FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | + FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) | + FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) | + FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) | + FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | + FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | + FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) | + FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) | + FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ set_64bit_val(wqe, IRDMA_BYTE_24, hdr); - if (info->push_wqe) { + if (info->push_wqe) irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); - } else { - if (post_sq) - irdma_uk_qp_post_wr(qp); - } + else if (post_sq) + irdma_uk_qp_post_wr(qp); return 0; } @@ -615,29 +607,45 @@ irdma_set_mw_bind_wqe_gen_1(__le64 * wqe, { set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)op_info->va); set_64bit_val(wqe, IRDMA_BYTE_8, - LS_64(op_info->mw_stag, IRDMAQPSQ_PARENTMRSTAG) | - LS_64(op_info->mr_stag, IRDMAQPSQ_MWSTAG)); + FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mw_stag) | + FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mr_stag)); set_64bit_val(wqe, IRDMA_BYTE_16, op_info->bind_len); } /** * irdma_copy_inline_data_gen_1 - Copy inline data to wqe - * @dest: pointer to wqe - * @src: pointer to inline data - * @len: length of inline data to copy + * @wqe: pointer to wqe + * @sge_list: table of pointers to inline data + * @num_sges: Total inline data length * @polarity: compatibility parameter */ static void -irdma_copy_inline_data_gen_1(u8 *dest, u8 *src, u32 len, - u8 polarity) +irdma_copy_inline_data_gen_1(u8 *wqe, struct irdma_sge *sge_list, + u32 num_sges, u8 polarity) { - if (len <= IRDMA_BYTE_16) { - irdma_memcpy(dest, src, len); - } else { - irdma_memcpy(dest, src, IRDMA_BYTE_16); - src += IRDMA_BYTE_16; *** 14298 LINES SKIPPED ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202212220114.2BM1ELBg040513>