Date: Tue, 28 Apr 2026 06:44:58 +0000 From: Sumit Saxena <ssaxena@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Cc: Chandrakanth Patil <chandrakanth.patil@broadcom.com> Subject: git: f2f831b2c151 - main - bnxt_en: Add core SR-IOV infrastructure Message-ID: <69f0576a.31521.3b0166c9@gitrepo.freebsd.org>
index | next in thread | raw e-mail
The branch main has been updated by ssaxena: URL: https://cgit.FreeBSD.org/src/commit/?id=f2f831b2c151a9d989a94fc7c894118c802ef348 commit f2f831b2c151a9d989a94fc7c894118c802ef348 Author: Chandrakanth Patil <chandrakanth.patil@broadcom.com> AuthorDate: 2026-03-31 16:59:00 +0000 Commit: Sumit Saxena <ssaxena@FreeBSD.org> CommitDate: 2026-04-28 06:16:16 +0000 bnxt_en: Add core SR-IOV infrastructure Introduce the foundational building blocks for SR-IOV Virtual Function support on Broadcom NetXtreme-C/E adapters. * Add bnxt_sriov.h: defines the extended bnxt_vf_info structure (per-VF firmware FID, MAC addresses, VLAN, flags, DMA command buffers, resource counts), the bnxt_resc_map helper, flag macros (BNXT_VF_TRUST, BNXT_VF_SPOOFCHK, etc.), and prototypes for all SR-IOV functions. * Add bnxt_sriov.c: implements the SR-IOV attachment sequence (bnxt_sriov_attach), the iflib IOV callbacks (bnxt_iov_init, bnxt_iov_uninit, bnxt_iov_vf_add), VF resource allocation and firmware configuration helpers (bnxt_alloc_vf_resources, bnxt_cfg_hw_sriov, bnxt_hwrm_func_vf_resc_cfg, bnxt_hwrm_func_buf_rgtr, bnxt_hwrm_func_vf_resource_free), and the per-VF parameter helper. * Extend bnxt.h: include bnxt_sriov.h; extend bnxt_pf_info with VF- tracking fields (vf array, firmware FID/MAC, resource-reservation strategy, DMA page management, sysctl context); replace the upstream bnxt_vf_info stub with the full definition from bnxt_sriov.h; extend bnxt_func_qcfg with allocation counters required by the VF resource configuration path; add vf_resc_cfg_input and sriov_lock to bnxt_softc. * Update Makefile to build bnxt_sriov.c and include bnxt_sriov.h. * Wire up PCI-IOV device methods (pci_iov_init / pci_iov_uninit / pci_iov_add_vf) and iflib IOV callbacks (ifdi_iov_init / ifdi_iov_uninit / ifdi_iov_vf_add) in if_bnxt.c; call bnxt_sriov_attach() from bnxt_attach_post() on P5+ Physical Functions. MFC after: 1 month Reviewed by: ssaxena Differential Revision: https://reviews.freebsd.org/D56197 --- sys/dev/bnxt/bnxt_en/bnxt.h | 55 ++--- sys/dev/bnxt/bnxt_en/bnxt_sriov.c | 469 ++++++++++++++++++++++++++++++++++++++ sys/dev/bnxt/bnxt_en/bnxt_sriov.h | 116 ++++++++++ sys/dev/bnxt/bnxt_en/if_bnxt.c | 19 +- sys/modules/bnxt/bnxt_en/Makefile | 1 + 5 files changed, 632 insertions(+), 28 deletions(-) diff --git a/sys/dev/bnxt/bnxt_en/bnxt.h b/sys/dev/bnxt/bnxt_en/bnxt.h index 64482a656e9d..bae419d0eefe 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt.h +++ b/sys/dev/bnxt/bnxt_en/bnxt.h @@ -47,6 +47,7 @@ #include "hsi_struct_def.h" #include "bnxt_dcb.h" #include "bnxt_auxbus_compat.h" +#include "bnxt_sriov.h" #define DFLT_HWRM_CMD_TIMEOUT 500 @@ -95,9 +96,18 @@ #define NETXTREME_C_VF1 0x16cb #define NETXTREME_C_VF2 0x16e1 #define NETXTREME_C_VF3 0x16e5 -#define NETXTREME_E_VF1 0x16c1 -#define NETXTREME_E_VF2 0x16d3 -#define NETXTREME_E_VF3 0x16dc + +#define NETXTREME_E_VF1 0x1606 +#define NETXTREME_E_VF2 0x1609 +#define NETXTREME_E_VF3 0x16c1 +#define NETXTREME_E_VF4 0x16d3 +#define NETXTREME_E_VF5 0x16dc + +#define NETXTREME_E_P5_VF1 0x1806 +#define NETXTREME_E_P5_VF2 0x1807 +#define NETXTREME_E_P5_VF_HV1 0x1808 +#define NETXTREME_E_P5_VF_HV2 0x1809 +#define NETXTREME_E_P7_VF 0x1819 #define EVENT_DATA1_RESET_NOTIFY_FATAL(data1) \ (((data1) & \ @@ -504,7 +514,9 @@ struct bnxt_pf_info { uint8_t port_id; uint32_t first_vf_id; uint16_t active_vfs; + uint16_t registered_vfs; uint16_t max_vfs; + uint16_t max_msix_vfs; uint32_t max_encap_records; uint32_t max_decap_records; uint32_t max_tx_em_flows; @@ -515,31 +527,14 @@ struct bnxt_pf_info { uint16_t hwrm_cmd_req_pages; void *hwrm_cmd_req_addr[4]; bus_addr_t hwrm_cmd_req_dma_addr[4]; -}; - -struct bnxt_vf_info { uint16_t fw_fid; uint8_t mac_addr[ETHER_ADDR_LEN]; - uint16_t max_rsscos_ctxs; - uint16_t max_cp_rings; - uint16_t max_tx_rings; - uint16_t max_rx_rings; - uint16_t max_hw_ring_grps; - uint16_t max_l2_ctxs; - uint16_t max_irqs; - uint16_t max_vnics; - uint16_t max_stat_ctxs; - uint32_t vlan; -#define BNXT_VF_QOS 0x1 -#define BNXT_VF_SPOOFCHK 0x2 -#define BNXT_VF_LINK_FORCED 0x4 -#define BNXT_VF_LINK_UP 0x8 - uint32_t flags; - uint32_t func_flags; /* func cfg flags */ - uint32_t min_tx_rate; - uint32_t max_tx_rate; - void *hwrm_cmd_req_addr; - bus_addr_t hwrm_cmd_req_dma_addr; + uint8_t vf_resv_strategy; + uint8_t num_vfs; + struct bnxt_vf_info *vf; + uint16_t vf_hwrm_cmd_req_page_shift; + struct sysctl_ctx_list sysctl_ctx; + struct sysctl_oid *sysctl_node; }; #define BNXT_PF(softc) (!((softc)->flags & BNXT_FLAG_VF)) @@ -698,6 +693,12 @@ struct bnxt_func_qcfg { uint16_t alloc_tx_rings; uint16_t alloc_rx_rings; uint16_t alloc_vnics; + uint16_t alloc_rss_ctx; + uint16_t alloc_l2_ctx; + uint16_t alloc_vfs; + uint16_t alloc_hw_ring_grps; + uint16_t alloc_stat_ctx; + uint16_t alloc_msix; }; struct bnxt_hw_lro { @@ -1089,6 +1090,7 @@ struct bnxt_softc { struct bnxt_func_qcfg fn_qcfg; struct bnxt_pf_info pf; struct bnxt_vf_info vf; + struct hwrm_func_vf_resource_cfg_input vf_resc_cfg_input; uint16_t hwrm_cmd_seq; uint32_t hwrm_cmd_timeo; /* milliseconds */ @@ -1097,6 +1099,7 @@ struct bnxt_softc { /* Interrupt info for HWRM */ struct if_irq irq; struct mtx hwrm_lock; + struct mtx sriov_lock; uint16_t hwrm_max_req_len; uint16_t hwrm_max_ext_req_len; uint32_t hwrm_spec_code; diff --git a/sys/dev/bnxt/bnxt_en/bnxt_sriov.c b/sys/dev/bnxt/bnxt_en/bnxt_sriov.c new file mode 100644 index 000000000000..b2a65c044cee --- /dev/null +++ b/sys/dev/bnxt/bnxt_en/bnxt_sriov.c @@ -0,0 +1,469 @@ +#include <linux/delay.h> +#include <linux/etherdevice.h> + +#include "opt_global.h" +#include "bnxt.h" +#include "hsi_struct_def.h" +#include "bnxt_hwrm.h" +#include "bnxt_sriov.h" + + +static int +bnxt_set_vf_admin_mac(struct bnxt_softc *softc, struct bnxt_vf_info *vf, + const uint8_t *mac) +{ + struct hwrm_func_cfg_input req = {0}; + int rc; + + if (!BNXT_PF(softc)) + return (EOPNOTSUPP); + + bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_CFG); + + req.fid = htole16(vf->fw_fid); + req.enables = htole32(HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_MAC_ADDR); + memcpy(req.dflt_mac_addr, mac, ETHER_ADDR_LEN); + + BNXT_HWRM_LOCK(softc); + rc = _hwrm_send_message(softc, &req, sizeof(req)); + BNXT_HWRM_UNLOCK(softc); + + return (rc); +} + +static void +bnxt_vf_parse_schema(struct bnxt_softc *softc, struct bnxt_vf_info *vf, + const nvlist_t *params) +{ + const void *mac; + size_t maclen; + + memset(vf->mac_addr, 0, ETHER_ADDR_LEN); + memset(vf->vf_mac_addr, 0, ETHER_ADDR_LEN); + + if (params == NULL) + return; + + if (nvlist_exists(params, "mac-anti-spoof")) + vf->spoofchk = nvlist_get_bool(params, "mac-anti-spoof"); + if (nvlist_exists(params, "trust")) + vf->trusted = nvlist_get_bool(params, "trust"); + + if (!nvlist_exists(params, "mac-addr")) + return; + + mac = nvlist_get_binary(params, "mac-addr", &maclen); + + if (maclen != ETHER_ADDR_LEN) + return; + + if (!is_valid_ether_addr(mac)) + return; + + memcpy(vf->mac_addr, mac, ETHER_ADDR_LEN); + vf->has_admin_mac = true; +} + +/* Add a Virtual Functions */ +int +bnxt_iov_vf_add(if_ctx_t ctx, uint16_t vfnum, const nvlist_t *params) +{ + struct bnxt_softc *softc = iflib_get_softc(ctx); + struct bnxt_vf_info *vf = &softc->pf.vf[vfnum]; + int rc; + + vf->fw_fid = softc->pf.first_vf_id + vfnum; + vf->vfnum = vfnum; + + /* Parse schema */ + bnxt_vf_parse_schema(softc, vf, params); + + /* + * If user provided MAC, program it into firmware. + */ + if (vf->has_admin_mac) { + rc = bnxt_set_vf_admin_mac(softc, vf, vf->mac_addr); + if (rc) + device_printf(softc->dev, + "vf%u: PF-assigned MAC programming failed (rc=%d), falling back to firmware/default MAC\n", + vfnum, rc); + } + + return 0; +} + +/* Free driver-side VF resources (called after hwrm_vf_resc_free) */ +void bnxt_free_vf_resources(struct bnxt_softc *softc) +{ + int i; + size_t page_size = 1UL << softc->pf.vf_hwrm_cmd_req_page_shift; + + softc->pf.active_vfs = 0; + + if (softc->pf.vf) { + kfree(softc->pf.vf); + softc->pf.vf = NULL; + } + if (softc->pf.vf_event_bmap) { + kfree(softc->pf.vf_event_bmap); + softc->pf.vf_event_bmap = NULL; + } + for (i = 0; i < softc->pf.hwrm_cmd_req_pages; i++) { + if (softc->pf.hwrm_cmd_req_addr[i]) { + dma_free_coherent(&softc->pdev->dev, page_size, + softc->pf.hwrm_cmd_req_addr[i], + softc->pf.hwrm_cmd_req_dma_addr[i]); + softc->pf.hwrm_cmd_req_addr[i] = NULL; + } + } +} + +/* Free firmware-side VF resources */ +int +bnxt_hwrm_func_vf_resource_free(struct bnxt_softc *softc, int num_vfs) +{ + int i, rc; + struct hwrm_func_vf_resc_free_input req; + + bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_VF_RESC_FREE); + + BNXT_HWRM_LOCK(softc); + for (i = softc->pf.first_vf_id; i < softc->pf.first_vf_id + num_vfs; i++) { + req.vf_id = cpu_to_le16(i); + rc = _hwrm_send_message(softc, &req, sizeof(req)); + if (rc) + break; + } + BNXT_HWRM_UNLOCK(softc); + + return rc; +} + +/* Free all VF resources */ +void bnxt_iov_uninit(if_ctx_t ctx) +{ + int rc; + struct bnxt_softc *softc = iflib_get_softc(ctx); + int num_vfs = softc->pf.num_vfs; + + if (!num_vfs) + return; + + BNXT_SRIOV_LOCK(softc); + softc->pf.num_vfs = 0; + BNXT_SRIOV_UNLOCK(softc); + + rc = bnxt_hwrm_func_vf_resource_free(softc, num_vfs); + if (rc) + device_printf(softc->dev, "VF resource free HWRM failed: %d\n", rc); + + bnxt_free_vf_resources(softc); + BNXT_SRIOV_LOCK_DESTROY(softc); +} + +static inline int +bnxt_set_vf_resc_field(uint16_t *min_field, uint16_t *max_field, + uint16_t hw_max, uint16_t pf_alloc, int num_vfs) +{ + uint16_t val = 0; + + if (num_vfs <= 0) + return -EINVAL; + + if (hw_max > pf_alloc) + val = (hw_max - pf_alloc) / num_vfs; + + *min_field = *max_field = cpu_to_le16(val); + + return 0; +} + +static int bnxt_set_vf_params(struct bnxt_softc *softc, int vf_id) +{ + struct hwrm_func_cfg_input req = {0}; + struct bnxt_vf_info *vf; + int rc; + + bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_CFG); + + vf = &softc->pf.vf[vf_id]; + req.fid = cpu_to_le16(vf->fw_fid); + + + if (is_valid_ether_addr(vf->mac_addr)) { + req.enables |= cpu_to_le32(HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_MAC_ADDR); + memcpy(req.dflt_mac_addr, vf->mac_addr, ETHER_ADDR_LEN); + } + + if (vf->vlan) { + req.enables |= cpu_to_le32(HWRM_FUNC_CFG_INPUT_ENABLES_DFLT_VLAN); + req.dflt_vlan = cpu_to_le16(vf->vlan); + } + + if (vf->flags & BNXT_VF_TRUST) + req.flags = cpu_to_le32(HWRM_FUNC_CFG_INPUT_FLAGS_TRUSTED_VF_ENABLE); + + BNXT_HWRM_LOCK(softc); + rc = _hwrm_send_message(softc, &req, sizeof(req)); + BNXT_HWRM_UNLOCK(softc); + if (rc) + device_printf(softc->dev, "hwrm_func_cfg failed (error:%d)\n", rc); + + return rc; +} + +static int +bnxt_hwrm_func_vf_resc_cfg(struct bnxt_softc *softc, int num_vfs, bool reset) +{ + struct hwrm_func_vf_resource_cfg_input req = {0}; + struct bnxt_pf_info *pf = &softc->pf; + struct bnxt_func_qcfg *fn_qcfg = &softc->fn_qcfg; + struct bnxt_hw_resc *hw_resc = &softc->hw_resc; + int i, rc; + uint16_t msix_val = 0; + + bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_VF_RESOURCE_CFG); + struct bnxt_resc_map resc_table[] = { + { &req.min_tx_rings, &req.max_tx_rings, hw_resc->max_tx_rings, fn_qcfg->alloc_tx_rings }, + { &req.min_rx_rings, &req.max_rx_rings, hw_resc->max_rx_rings, fn_qcfg->alloc_rx_rings }, + { &req.min_cmpl_rings, &req.max_cmpl_rings, hw_resc->max_cp_rings, fn_qcfg->alloc_completion_rings }, + { &req.min_stat_ctx, &req.max_stat_ctx, hw_resc->max_stat_ctxs, fn_qcfg->alloc_stat_ctx }, + { &req.min_vnics, &req.max_vnics, hw_resc->max_vnics, fn_qcfg->alloc_vnics }, + { &req.min_hw_ring_grps, &req.max_hw_ring_grps, hw_resc->max_hw_ring_grps, fn_qcfg->alloc_hw_ring_grps }, + { &req.min_rsscos_ctx, &req.max_rsscos_ctx, hw_resc->max_rsscos_ctxs, fn_qcfg->alloc_rss_ctx }, + { &req.min_l2_ctxs, &req.max_l2_ctxs, hw_resc->max_l2_ctxs, fn_qcfg->alloc_l2_ctx }, + }; + + for (i = 0; i < sizeof(resc_table) / sizeof(resc_table[0]); i++) { + rc = bnxt_set_vf_resc_field(resc_table[i].min_field, + resc_table[i].max_field, + resc_table[i].hw_max, + resc_table[i].pf_alloc, + num_vfs); + if (rc) + return rc; + } + + if (hw_resc->max_irqs > fn_qcfg->alloc_msix && num_vfs > 0) + msix_val = (hw_resc->max_irqs - fn_qcfg->alloc_msix) / num_vfs; + + req.max_msix = cpu_to_le16(msix_val); + + for (i = 0; i < num_vfs; i++) { + struct bnxt_vf_info *vf = &pf->vf[i]; + + vf->fw_fid = pf->first_vf_id + i; + if (reset) { + rc = bnxt_set_vf_params(softc, i); + if (rc) + break; + } + + req.vf_id = cpu_to_le16(vf->fw_fid); + + BNXT_HWRM_LOCK(softc); + rc = _hwrm_send_message(softc, &req, sizeof(req)); + BNXT_HWRM_UNLOCK(softc); + + if (rc) { + device_printf(softc->dev, "HWRM_FUNC_VF_RESOURCE_CFG req dump:\n"); + break; + } + + pf->active_vfs = i + 1; + + vf->min_tx_rings = le16_to_cpu(req.min_tx_rings); + vf->min_rx_rings = le16_to_cpu(req.min_rx_rings); + vf->min_cp_rings = le16_to_cpu(req.min_cmpl_rings); + vf->min_stat_ctxs = le16_to_cpu(req.min_stat_ctx); + vf->min_ring_grps = le16_to_cpu(req.min_hw_ring_grps); + vf->min_vnics = le16_to_cpu(req.min_vnics); + } + + if (pf->active_vfs) + memcpy(&softc->vf_resc_cfg_input, &req, + sizeof(struct hwrm_func_vf_resource_cfg_input)); + + return rc; +} + +static int +bnxt_hwrm_func_buf_rgtr(struct bnxt_softc *softc) +{ + int rc; + struct hwrm_func_buf_rgtr_input req; + + bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_BUF_RGTR); + + req.req_buf_num_pages = cpu_to_le16(softc->pf.hwrm_cmd_req_pages); + req.req_buf_page_size = cpu_to_le16(softc->pf.vf_hwrm_cmd_req_page_shift); + req.req_buf_len = cpu_to_le16(BNXT_HWRM_REQ_MAX_SIZE); + req.req_buf_page_addr0 = cpu_to_le64(softc->pf.hwrm_cmd_req_dma_addr[0]); + req.req_buf_page_addr1 = cpu_to_le64(softc->pf.hwrm_cmd_req_dma_addr[1]); + req.req_buf_page_addr2 = cpu_to_le64(softc->pf.hwrm_cmd_req_dma_addr[2]); + req.req_buf_page_addr3 = cpu_to_le64(softc->pf.hwrm_cmd_req_dma_addr[3]); + + BNXT_HWRM_LOCK(softc); + rc = _hwrm_send_message(softc, &req, sizeof(req)); + BNXT_HWRM_UNLOCK(softc); + + return rc; +} + +static void +bnxt_set_vf_attr(struct bnxt_softc *softc, int num_vfs) +{ + int i; + struct bnxt_vf_info *vf; + + for (i = 0; i < num_vfs; i++) { + vf = &softc->pf.vf[i]; + memset(vf, 0, sizeof(*vf)); + } +} + +static int +bnxt_alloc_vf_resources(struct bnxt_softc *softc, int num_vfs) +{ + struct pci_dev *pdev = softc->pdev; + u32 nr_pages, size, i, j, k = 0; + u32 page_size, reqs_per_page; + void *p; + + p = kcalloc(num_vfs, sizeof(struct bnxt_vf_info), GFP_KERNEL); + if (!p) + return ENOMEM; + + rcu_assign_pointer(softc->pf.vf, p); + bnxt_set_vf_attr(softc, num_vfs); + + size = num_vfs * BNXT_HWRM_REQ_MAX_SIZE; + page_size = BNXT_PAGE_SIZE; + softc->pf.vf_hwrm_cmd_req_page_shift = BNXT_PAGE_SHIFT; + while (size > page_size * BNXT_MAX_VF_CMD_FWD_PAGES) { + page_size *= 2; + softc->pf.vf_hwrm_cmd_req_page_shift++; + } + nr_pages = DIV_ROUND_UP(size, page_size); + reqs_per_page = page_size / BNXT_HWRM_REQ_MAX_SIZE; + + for (i = 0; i < nr_pages; i++) { + softc->pf.hwrm_cmd_req_addr[i] = + dma_alloc_coherent(&pdev->dev, page_size, + &softc->pf.hwrm_cmd_req_dma_addr[i], + GFP_ATOMIC); + + if (!softc->pf.hwrm_cmd_req_addr[i]) + return ENOMEM; + + for (j = 0; j < reqs_per_page && k < num_vfs; j++) { + struct bnxt_vf_info *vf = &softc->pf.vf[k]; + + vf->hwrm_cmd_req_addr = (char *)softc->pf.hwrm_cmd_req_addr[i] + + j * BNXT_HWRM_REQ_MAX_SIZE; + vf->hwrm_cmd_req_dma_addr = + softc->pf.hwrm_cmd_req_dma_addr[i] + j * + BNXT_HWRM_REQ_MAX_SIZE; + k++; + } + } + + softc->pf.vf_event_bmap = kzalloc(ALIGN(DIV_ROUND_UP(num_vfs, 8), + sizeof(long)), GFP_ATOMIC); + if (!softc->pf.vf_event_bmap) + return ENOMEM; + + softc->pf.hwrm_cmd_req_pages = nr_pages; + + return 0; +} + +int bnxt_cfg_hw_sriov(struct bnxt_softc *softc, uint16_t *num_vfs, bool reset) +{ + int rc; + + rc = bnxt_hwrm_func_buf_rgtr(softc); + if (rc) { + device_printf(softc->dev, "hwrm func buf rgtr failed (error=%d)\n", rc); + return (EIO); + } + + rc = bnxt_hwrm_func_vf_resc_cfg(softc, *num_vfs, reset); + if (rc) { + device_printf(softc->dev, "hwrm func VF resc config failed (error=%d)\n", rc); + return (EIO); + } + + return (0); +} + +int +bnxt_iov_init(if_ctx_t ctx, uint16_t num_vfs, const nvlist_t *params) +{ + int rc; + if_t ifp = iflib_get_ifp(ctx); + struct bnxt_softc *softc = iflib_get_softc(ctx); + bool admin_up = !!(if_getflags(ifp) & IFF_UP); + bool running = !!(if_getdrvflags(ifp) & IFF_DRV_RUNNING); + + if (!admin_up || !running) { + device_printf(softc->dev, "PF is down, rejecting VF creation\n"); + return ENETDOWN; + } + + if (num_vfs > BNXT_MAX_VFS) { + device_printf(softc->dev, "Requested %u VFs exceeds maximum supported (%u)\n", + num_vfs, BNXT_MAX_VFS); + return ERANGE; + } + + /* + * Initialize SR-IOV lock before creating any SR-IOV state, so sysctl/VF + * paths can safely synchronize and error paths can always destroy it. + */ + BNXT_SRIOV_LOCK_INIT(softc, device_get_nameunit(softc->dev)); + + rc = bnxt_alloc_vf_resources(softc, num_vfs); + if (rc) { + device_printf(softc->dev, "VF resource alloc failed (error=%d)\n", rc); + goto fail_lock; + } + + rc = bnxt_cfg_hw_sriov(softc, &num_vfs, false); + if (rc) + goto fail_free_vf_resc; + + + BNXT_SRIOV_LOCK(softc); + softc->pf.num_vfs = num_vfs; + BNXT_SRIOV_UNLOCK(softc); + + return 0; + +fail_free_vf_resc: + bnxt_free_vf_resources(softc); +fail_lock: + BNXT_SRIOV_LOCK_DESTROY(softc); + return rc; +} + +void bnxt_sriov_attach(struct bnxt_softc *softc) +{ + int rc; + device_t dev = softc->dev; + nvlist_t *pf_schema, *vf_schema; + + pf_schema = pci_iov_schema_alloc_node(); + vf_schema = pci_iov_schema_alloc_node(); + + /* Optionally add VF-specific attributes to the VF schema */ + pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL); + pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof", IOV_SCHEMA_HASDEFAULT, FALSE); + pci_iov_schema_add_bool(vf_schema, "trust", IOV_SCHEMA_HASDEFAULT, FALSE); + + /* Attach SR-IOV schemas to the device */ + rc = pci_iov_attach(dev, pf_schema, vf_schema); + if (rc) + device_printf(dev, "Failed to initialize SR-IOV (error=%d)\n", rc); +} + diff --git a/sys/dev/bnxt/bnxt_en/bnxt_sriov.h b/sys/dev/bnxt/bnxt_en/bnxt_sriov.h new file mode 100644 index 000000000000..c0c3d2d70b08 --- /dev/null +++ b/sys/dev/bnxt/bnxt_en/bnxt_sriov.h @@ -0,0 +1,116 @@ +#ifndef _BNXT_SRIOV_H_ +#define _BNXT_SRIOV_H_ + +#include <sys/iov_schema.h> +#include <linux/pci.h> +#include <dev/pci/pci_iov.h> + +#include "opt_global.h" +#include "bnxt.h" + +#ifndef PCI_IOV +#define PCI_IOV 1 +#endif + +/* macro definations */ + +#define BNXT_MAX_VFS 4 +#define BNXT_HWRM_REQ_MAX_SIZE 128 +#define BNXT_MAX_VF_CMD_FWD_PAGES 4 +#define BNXT_VF_QOS 0x1 +#define BNXT_VF_SPOOFCHK 0x2 +#define BNXT_VF_LINK_FORCED 0x4 +#define BNXT_VF_LINK_UP 0x8 +#define BNXT_VF_TRUST 0x10 +#define BNXT_VLAN_VID_MASK 0x0fff + +#define BNXT_EXEC_FWD_RESP_SIZE_ERR(n) \ + ((offsetof(struct hwrm_exec_fwd_resp_input, encap_request) + n) >\ + offsetof(struct hwrm_exec_fwd_resp_input, encap_resp_target_id)) + +#define BNXT_VF_RESV_STRATEGY_MAXIMAL 0 +#define BNXT_VF_RESV_STRATEGY_MINIMAL 1 +#define BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC 2 +#define FUNC_RESOURCE_QCAPS_RESP_FLAGS_MIN_GUARANTEED 0x1UL + +#define BNXT_SRIOV_LOCK_INIT(sc, _name) \ + mtx_init(&(sc)->sriov_lock, _name, "sriov_lock", MTX_DEF | MTX_NOWITNESS) +#define BNXT_SRIOV_LOCK(sc) mtx_lock(&(sc)->sriov_lock) +#define BNXT_SRIOV_UNLOCK(sc) mtx_unlock(&(sc)->sriov_lock) +#define BNXT_SRIOV_LOCK_DESTROY(sc) \ + do { \ + if (mtx_initialized(&(sc)->sriov_lock)) \ + mtx_destroy(&(sc)->sriov_lock); \ +} while (0) + + +/* structure declartions/definations */ + +struct bnxt_softc; + +struct bnxt_vf_info { + uint8_t vfnum; + uint16_t fw_fid; + uint8_t mac_addr[ETHER_ADDR_LEN]; + uint8_t vf_mac_addr[ETHER_ADDR_LEN]; + uint32_t vlan; + uint32_t flags; + uint32_t func_qcfg_flags; + uint32_t min_tx_rate; + uint32_t max_tx_rate; + uint16_t min_tx_rings; + uint16_t max_tx_rings; + uint16_t min_rx_rings; + uint16_t max_rx_rings; + uint16_t min_cp_rings; + uint16_t max_cp_rings; + uint16_t min_rsscos_ctxs; + uint16_t max_rsscos_ctxs; + uint16_t min_stat_ctxs; + uint16_t max_stat_ctxs; + uint16_t min_ring_grps; + uint16_t max_hw_ring_grps; + uint16_t min_vnics; + uint16_t max_vnics; + uint16_t min_irqs; + uint16_t max_irqs; + uint16_t min_l2_ctxs; + uint16_t max_l2_ctxs; + void *hwrm_cmd_req_addr; + bus_addr_t hwrm_cmd_req_dma_addr; + struct iflib_dma_info hwrm_cmd_req; + uint16_t trusted; + bool spoofchk; + bool has_admin_mac; +}; + +struct bnxt_resc_map { + uint16_t *min_field; + uint16_t *max_field; + uint16_t hw_max; + uint16_t pf_alloc; +}; + +/* function prototypes */ + +void bnxt_sriov_attach(struct bnxt_softc *softc); +int bnxt_iov_init(if_ctx_t ctx, uint16_t num_vfs, const nvlist_t *params); +void bnxt_iov_uninit(if_ctx_t ctx); +int bnxt_iov_vf_add(if_ctx_t ctx, uint16_t vfnum, const nvlist_t *params); +int bnxt_hwrm_func_vf_resource_free(struct bnxt_softc *softc, int num_vfs); +void bnxt_free_vf_resources(struct bnxt_softc *softc); +int bnxt_create_trusted_vf_sysctls(struct bnxt_softc *softc, uint16_t num_vfs); +int bnxt_create_spoofchk_vf_sysctls(struct bnxt_softc *softc, uint16_t num_vfs); +bool bnxt_is_trusted_vf(struct bnxt_softc *bp, struct bnxt_vf_info *vf); +void bnxt_hwrm_exec_fwd_req(struct bnxt_softc *bp); +void bnxt_destroy_trusted_vf_sysctls(struct bnxt_softc *softc); +int bnxt_set_vf_trust(struct bnxt_softc *softc, int vf_id, bool trusted); +int bnxt_approve_mac(struct bnxt_softc *sc); +void bnxt_update_vf_mac(struct bnxt_softc *sc); +bool bnxt_promisc_ok(struct bnxt_softc *softc); +int bnxt_set_vf_spoofchk(struct bnxt_softc *sc, int vf_id, bool enable); +int bnxt_cfg_hw_sriov(struct bnxt_softc *softc, uint16_t *num_vfs, bool reset); +void bnxt_reenable_sriov(struct bnxt_softc *bp); + + +#endif /* _BNXT_SRIOV_H_ */ diff --git a/sys/dev/bnxt/bnxt_en/if_bnxt.c b/sys/dev/bnxt/bnxt_en/if_bnxt.c index dea6fd68181e..b9fb7fb8a7e4 100644 --- a/sys/dev/bnxt/bnxt_en/if_bnxt.c +++ b/sys/dev/bnxt/bnxt_en/if_bnxt.c @@ -266,6 +266,11 @@ static device_method_t bnxt_methods[] = { DEVMETHOD(device_shutdown, iflib_device_shutdown), DEVMETHOD(device_suspend, iflib_device_suspend), DEVMETHOD(device_resume, iflib_device_resume), +#ifdef PCI_IOV + DEVMETHOD(pci_iov_init, iflib_device_iov_init), + DEVMETHOD(pci_iov_uninit, iflib_device_iov_uninit), + DEVMETHOD(pci_iov_add_vf, iflib_device_iov_add_vf), +#endif DEVMETHOD_END }; @@ -344,7 +349,11 @@ static device_method_t bnxt_iflib_methods[] = { DEVMETHOD(ifdi_i2c_req, bnxt_i2c_req), DEVMETHOD(ifdi_needs_restart, bnxt_if_needs_restart), - +#ifdef PCI_IOV + DEVMETHOD(ifdi_iov_init, bnxt_iov_init), + DEVMETHOD(ifdi_iov_uninit, bnxt_iov_uninit), + DEVMETHOD(ifdi_iov_vf_add, bnxt_iov_vf_add), +#endif DEVMETHOD_END }; @@ -2738,6 +2747,12 @@ bnxt_attach_post(if_ctx_t ctx) bnxt_dcb_init(softc); bnxt_rdma_aux_device_init(softc); +#if PCI_IOV + /* SR-IOV attach */ + if (BNXT_PF(softc) && BNXT_CHIP_P5_PLUS(softc)) + bnxt_sriov_attach(softc); +#endif + failed: return rc; } @@ -5324,4 +5339,4 @@ bnxt_get_wol_settings(struct bnxt_softc *softc) do { wol_handle = bnxt_hwrm_get_wol_fltrs(softc, wol_handle); } while (wol_handle && wol_handle != BNXT_NO_MORE_WOL_FILTERS); -} +} \ No newline at end of file diff --git a/sys/modules/bnxt/bnxt_en/Makefile b/sys/modules/bnxt/bnxt_en/Makefile index 12c23898c106..dc9b30b7f2af 100644 --- a/sys/modules/bnxt/bnxt_en/Makefile +++ b/sys/modules/bnxt/bnxt_en/Makefile @@ -9,6 +9,7 @@ SRCS += bnxt_sysctl.c SRCS += bnxt_mgmt.c SRCS += bnxt_dcb.c bnxt_dcb.h SRCS += bnxt_auxbus_compat.c bnxt_auxbus_compat.h +SRCS += bnxt_sriov.c bnxt_sriov.h SRCS += bnxt_ulp.c bnxt_ulp.h SRCS += ${LINUXKPI_GENSRCS}home | help
Want to link to this message? Use this
URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?69f0576a.31521.3b0166c9>