Date: Fri, 07 Aug 2026 07:51:57 +0000 From: Kevin Bowling <kbowling@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 86199f1a74ab - main - ixgbe: restart iflib around SR-IOV reconfiguration Message-ID: <6a758e9d.43421.42b881c4@gitrepo.freebsd.org>
index | next in thread | raw e-mail
The branch main has been updated by kbowling: URL: https://cgit.FreeBSD.org/src/commit/?id=86199f1a74abc76c3bb1ed15ccee7df3cabf3d7b commit 86199f1a74abc76c3bb1ed15ccee7df3cabf3d7b Author: Kevin Bowling <kbowling@FreeBSD.org> AuthorDate: 2026-08-06 11:03:35 +0000 Commit: Kevin Bowling <kbowling@FreeBSD.org> CommitDate: 2026-08-07 07:51:35 +0000 ixgbe: restart iflib around SR-IOV reconfiguration The IOV callback changes the PF pool, virtualization mode, and hardware queue indices while iflib still considers the old queue layout live. Teardown likewise leaves the software pool and mode at their SR-IOV values. Use iflib stop/mutate/restart transactions for both transitions. Disable VF DMA and PCI VF Enable before queue reuse, let outstanding transactions drain, and restore the non-IOV pool and queue indices on teardown. Remove the redundant driver-local pci_iov_detach() wrapper; iflib already performs that check centrally before the driver detach callback. It may be possible to avoid some restart in the future on this hardware pausing DMA and remapping rings but not pursued yet. MFC after: 2 weeks --- sys/dev/ixgbe/if_ix.c | 10 ++-------- sys/dev/ixgbe/if_sriov.c | 36 +++++++++++++++++++++++++----------- sys/dev/ixgbe/ixgbe_sriov.h | 2 -- 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/sys/dev/ixgbe/if_ix.c b/sys/dev/ixgbe/if_ix.c index fe9d14fde2f9..38029e3ec895 100644 --- a/sys/dev/ixgbe/if_ix.c +++ b/sys/dev/ixgbe/if_ix.c @@ -297,8 +297,8 @@ static device_method_t ix_methods[] = { DEVMETHOD(device_suspend, iflib_device_suspend), DEVMETHOD(device_resume, iflib_device_resume), #ifdef PCI_IOV - DEVMETHOD(pci_iov_init, iflib_device_iov_init), - DEVMETHOD(pci_iov_uninit, iflib_device_iov_uninit), + DEVMETHOD(pci_iov_init, iflib_device_iov_init_restart), + DEVMETHOD(pci_iov_uninit, iflib_device_iov_uninit_restart), DEVMETHOD(pci_iov_add_vf, iflib_device_iov_add_vf), #endif /* PCI_IOV */ DEVMETHOD(bus_add_child, device_add_child_ordered), @@ -3665,16 +3665,10 @@ static int ixgbe_if_detach(if_ctx_t ctx) { struct ixgbe_softc *sc = iflib_get_softc(ctx); - device_t dev = iflib_get_dev(ctx); u32 ctrl_ext; INIT_DEBUGOUT("ixgbe_detach: begin"); - if (ixgbe_pci_iov_detach(dev) != 0) { - device_printf(dev, "SR-IOV in use; detach first.\n"); - return (EBUSY); - } - ixgbe_setup_low_power_mode(ctx); /* let hardware know driver is unloading */ diff --git a/sys/dev/ixgbe/if_sriov.c b/sys/dev/ixgbe/if_sriov.c index 31db2cbb0407..c05b82533081 100644 --- a/sys/dev/ixgbe/if_sriov.c +++ b/sys/dev/ixgbe/if_sriov.c @@ -41,15 +41,6 @@ MALLOC_DEFINE(M_IXGBE_SRIOV, "ix_sriov", "ix SR-IOV allocations"); -/************************************************************************ - * ixgbe_pci_iov_detach - ************************************************************************/ -int -ixgbe_pci_iov_detach(device_t dev) -{ - return pci_iov_detach(dev); -} - /************************************************************************ * ixgbe_define_iov_schemas ************************************************************************/ @@ -1098,7 +1089,6 @@ ixgbe_if_iov_init(if_ctx_t ctx, u16 num_vfs, const nvlist_t *config) ixgbe_init_mbx_params_pf(&sc->hw); sc->feat_en |= IXGBE_FEATURE_SRIOV; - ixgbe_if_init(sc->ctx); return (retval); @@ -1116,7 +1106,8 @@ ixgbe_if_iov_uninit(if_ctx_t ctx) struct ixgbe_hw *hw; struct ixgbe_softc *sc; uint32_t pf_reg, vf_reg; - int i; + int error, i, iov_pos; + u16 iov_ctl; sc = iflib_get_softc(ctx); hw = &sc->hw; @@ -1132,6 +1123,26 @@ ixgbe_if_iov_uninit(if_ctx_t ctx) vf_reg = 0; IXGBE_WRITE_REG(hw, IXGBE_VFRE(vf_reg), 0); IXGBE_WRITE_REG(hw, IXGBE_VFTE(vf_reg), 0); + IXGBE_WRITE_FLUSH(hw); + + /* + * pci_iov(4) normally clears VF Enable after this callback returns, + * but iflib's restart transaction reuses the PF queues first. Disable + * the VFs here and allow outstanding transactions to drain before the + * queue layout changes. + */ + error = pci_find_extcap(sc->dev, PCIZ_SRIOV, &iov_pos); + if (error == 0) { + iov_ctl = pci_read_config(sc->dev, + iov_pos + PCIR_SRIOV_CTL, 2); + iov_ctl &= ~(PCIM_SRIOV_VF_EN | PCIM_SRIOV_VF_MSE); + pci_write_config(sc->dev, iov_pos + PCIR_SRIOV_CTL, + iov_ctl, 2); + pause("ixiov", MAX(1, howmany(hz, 10))); + } else + device_printf(sc->dev, + "could not disable PCI SR-IOV before queue reuse: %d\n", + error); for (i = 0; i < sc->num_vfs; i++) { if (!(sc->vfs[i].flags & IXGBE_VF_ACTIVE)) @@ -1155,6 +1166,9 @@ ixgbe_if_iov_uninit(if_ctx_t ctx) free(sc->vfs, M_IXGBE_SRIOV); sc->vfs = NULL; sc->feat_en &= ~IXGBE_FEATURE_SRIOV; + sc->pool = 0; + sc->iov_mode = IXGBE_NO_VM; + ixgbe_align_all_queue_indices(sc); sc->iov_vfta_valid = false; sc->iov_vlan_promisc = false; (void)ixgbe_clear_vfta(hw); diff --git a/sys/dev/ixgbe/ixgbe_sriov.h b/sys/dev/ixgbe/ixgbe_sriov.h index 84b5e25a8c3c..c38f4075b97a 100644 --- a/sys/dev/ixgbe/ixgbe_sriov.h +++ b/sys/dev/ixgbe/ixgbe_sriov.h @@ -79,7 +79,6 @@ void ixgbe_initialize_iov(struct ixgbe_softc *); void ixgbe_recalculate_max_frame(struct ixgbe_softc *); void ixgbe_ping_all_vfs(struct ixgbe_softc *); u_int ixgbe_iov_rebuild_mta(struct ixgbe_softc *); -int ixgbe_pci_iov_detach(device_t); void ixgbe_define_iov_schemas(device_t, int *); void ixgbe_align_all_queue_indices(struct ixgbe_softc *); int ixgbe_vf_que_index(int, int, int); @@ -96,7 +95,6 @@ u32 ixgbe_get_mrqc(int); #define ixgbe_initialize_iov(_a) #define ixgbe_recalculate_max_frame(_a) #define ixgbe_ping_all_vfs(_a) -#define ixgbe_pci_iov_detach(_a) 0 #define ixgbe_define_iov_schemas(_a,_b) #define ixgbe_align_all_queue_indices(_a) #define ixgbe_vf_que_index(_a, _b, _c) (_c)home | help
Want to link to this message? Use this
URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?6a758e9d.43421.42b881c4>