Date: Fri, 07 Aug 2026 14:43:07 +0000 From: Kevin Bowling <kbowling@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: aea4240ef583 - main - ixgbe: Quiesce VFs across PF reset Message-ID: <6a75eefb.47079.78db7538@gitrepo.freebsd.org>
index | next in thread | raw e-mail
The branch main has been updated by kbowling: URL: https://cgit.FreeBSD.org/src/commit/?id=aea4240ef5834fb4a47f80c659c80f902cb4bb06 commit aea4240ef5834fb4a47f80c659c80f902cb4bb06 Author: Kevin Bowling <kbowling@FreeBSD.org> AuthorDate: 2026-08-06 08:21:45 +0000 Commit: Kevin Bowling <kbowling@FreeBSD.org> CommitDate: 2026-08-07 14:38:12 +0000 ixgbe: Quiesce VFs across PF reset Stop VF transmit and receive in hardware, clear PF-side mailbox CTS, and notify active VFs before resetting a PF. A PF reset invalidates VF queue state, so the no-CTS control message makes cooperative VFs discard stale state and renegotiate after the PF returns. The hardware queue gates synchronously prevent further VF DMA. Do not hold the exclusive iflib context lock for a fixed VF-watchdog interval after the reset. Report the PF link transition directly instead of dispatching mailbox work from the stop path, which could otherwise re-enable VF I/O mid-reset. The CTS, PF-control, and VF queue controls follow the reset mechanisms used by DPDK. MFC after: 2 weeks --- sys/dev/ixgbe/if_ix.c | 11 +++++++++-- sys/dev/ixgbe/if_sriov.c | 34 ++++++++++++++++++++++++++++++++++ sys/dev/ixgbe/ixgbe_sriov.h | 2 ++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/sys/dev/ixgbe/if_ix.c b/sys/dev/ixgbe/if_ix.c index 5ab30a8054a9..e0e627b915d9 100644 --- a/sys/dev/ixgbe/if_ix.c +++ b/sys/dev/ixgbe/if_ix.c @@ -4757,8 +4757,10 @@ ixgbe_if_stop(if_ctx_t ctx) INIT_DEBUGOUT("ixgbe_if_stop: begin\n"); - if (sc->feat_en & IXGBE_FEATURE_SRIOV) + if (sc->feat_en & IXGBE_FEATURE_SRIOV) { ixgbe_disable_mdd(hw); + ixgbe_quiesce_vfs(sc); + } ixgbe_reset_hw(hw); hw->adapter_stopped = false; ixgbe_stop_adapter(hw); @@ -4767,7 +4769,12 @@ ixgbe_if_stop(if_ctx_t ctx) /* Update the stack */ sc->link_up = false; - ixgbe_if_update_admin_status(ctx); + if (sc->link_active) { + if (bootverbose) + device_printf(sc->dev, "Link is Down\n"); + iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); + sc->link_active = false; + } /* reprogram the RAR[0] in case user changed it. */ ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); diff --git a/sys/dev/ixgbe/if_sriov.c b/sys/dev/ixgbe/if_sriov.c index af0027a073c8..8611f2685611 100644 --- a/sys/dev/ixgbe/if_sriov.c +++ b/sys/dev/ixgbe/if_sriov.c @@ -283,6 +283,40 @@ ixgbe_ping_all_vfs(struct ixgbe_softc *sc) } } /* ixgbe_ping_all_vfs */ +/* + * Stop VF DMA before resetting the PF. A PF reset invalidates the VF queue + * state, so allowing an active VF to resume with its old rings can strand + * descriptors in both the VF and PF. Clearing CTS makes a cooperative VF + * renegotiate its state after the PF comes back; it is deliberately separate + * from IXGBE_VF_IO_DISABLED, which records a persistent administrative or + * recovery decision. + */ +void +ixgbe_quiesce_vfs(struct ixgbe_softc *sc) +{ + struct ixgbe_hw *hw; + struct ixgbe_vf *vf; + uint32_t index, mask, vfre, vfte; + int i; + + hw = &sc->hw; + for (i = 0; i < sc->num_vfs; i++) { + vf = &sc->vfs[i]; + if (!(vf->flags & IXGBE_VF_ACTIVE)) + continue; + + vf->flags &= ~IXGBE_VF_CTS; + index = IXGBE_VF_INDEX(vf->pool); + mask = IXGBE_VF_BIT(vf->pool); + vfte = IXGBE_READ_REG(hw, IXGBE_VFTE(index)); + vfre = IXGBE_READ_REG(hw, IXGBE_VFRE(index)); + IXGBE_WRITE_REG(hw, IXGBE_VFTE(index), vfte & ~mask); + IXGBE_WRITE_REG(hw, IXGBE_VFRE(index), vfre & ~mask); + ixgbe_send_vf_msg(hw, vf, IXGBE_PF_CONTROL_MSG); + } + IXGBE_WRITE_FLUSH(hw); +} /* ixgbe_quiesce_vfs */ + static bool ixgbe_pf_owns_vlan(struct ixgbe_softc *sc, uint16_t tag) diff --git a/sys/dev/ixgbe/ixgbe_sriov.h b/sys/dev/ixgbe/ixgbe_sriov.h index dbe56ec1171c..de0485f6e9a2 100644 --- a/sys/dev/ixgbe/ixgbe_sriov.h +++ b/sys/dev/ixgbe/ixgbe_sriov.h @@ -89,6 +89,7 @@ int ixgbe_iov_validate(struct ixgbe_softc *, u16); void ixgbe_if_iov_uninit(if_ctx_t); void ixgbe_initialize_iov(struct ixgbe_softc *); void ixgbe_activate_vfs(struct ixgbe_softc *); +void ixgbe_quiesce_vfs(struct ixgbe_softc *); void ixgbe_recalculate_max_frame(struct ixgbe_softc *); void ixgbe_ping_all_vfs(struct ixgbe_softc *); void ixgbe_init_iov_recovery(struct ixgbe_softc *); @@ -109,6 +110,7 @@ u32 ixgbe_get_mrqc(int); #define ixgbe_uninit_iov(_a) #define ixgbe_initialize_iov(_a) #define ixgbe_activate_vfs(_a) +#define ixgbe_quiesce_vfs(_a) #define ixgbe_recalculate_max_frame(_a) #define ixgbe_ping_all_vfs(_a) #define ixgbe_init_iov_recovery(_a)home | help
Want to link to this message? Use this
URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?6a75eefb.47079.78db7538>