Date: Fri, 07 Aug 2026 13:46:44 +0000 From: Kevin Bowling <kbowling@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 31285bddf21e - main - ixgbe: quarantine repeatedly faulting legacy VFs Message-ID: <6a75e1c4.41234.42acfc7f@gitrepo.freebsd.org>
index | next in thread | raw e-mail
The branch main has been updated by kbowling: URL: https://cgit.FreeBSD.org/src/commit/?id=31285bddf21e986e006e5405acc2c4626e43d190 commit 31285bddf21e986e006e5405acc2c4626e43d190 Author: Kevin Bowling <kbowling@FreeBSD.org> AuthorDate: 2026-08-06 11:05:13 +0000 Commit: Kevin Bowling <kbowling@FreeBSD.org> CommitDate: 2026-08-07 13:46:36 +0000 ixgbe: quarantine repeatedly faulting legacy VFs A guest can reinitialize after a VF function-level reset and repeatedly strand an 82599 or X540 PF with invalid descriptor DMA targets. Count only distinct Received Master Abort events accepted by the qualified transmit-stall detector and quarantine the VF after five events. Preserve quarantine across PF reinitialization, reject reset mailbox requests, and keep transmit, receive, and clear-to-send disabled. Recreating SR-IOV clears quarantine. Expose the affected pools through a read-only bitmap. After a successful quarantine FLR, leave the function in post-FLR configuration, explicitly keep decode and bus mastering disabled, verify the Command register, and refresh its PCI-layer cache so a later restore cannot re-enable the function. This addresses CVE-2021-33061 on 82599. Apply the same bounded-failure policy to X540 as defense in depth; the CVE does not list X540. Intel documents the 82599 issue in: http://iommu.com/datasheets/ethernet/controllers-nics/intel/ixgbe/Intel_82599_Application_Note_655276.pdf MFC after: 2 weeks Security: CVE-2021-33061 --- share/man/man4/ix.4 | 7 +++++ sys/dev/ixgbe/if_ix.c | 8 +++++ sys/dev/ixgbe/if_sriov.c | 72 ++++++++++++++++++++++++++++++++++++--------- sys/dev/ixgbe/ixgbe.h | 3 ++ sys/dev/ixgbe/ixgbe_sriov.h | 4 ++- 5 files changed, 79 insertions(+), 15 deletions(-) diff --git a/share/man/man4/ix.4 b/share/man/man4/ix.4 index 275e953cda2d..822a8040caf4 100644 --- a/share/man/man4/ix.4 +++ b/share/man/man4/ix.4 @@ -112,6 +112,13 @@ receive paths disabled until the VF completes a new reset handshake. On 82599 and X540 devices, the driver instead watches for transmit stalls caused by an invalid VF DMA target and resets the VF identified by its PCIe error status. +After five such events from the same VF, the driver quarantines that VF and +keeps its transmit, receive, and mailbox clear-to-send state disabled. +Quarantine persists across PF reinitialization and is cleared when the SR-IOV +configuration is destroyed and recreated. +The read-only +.Va dev.ix.N.iov_quarantined_vfs +sysctl reports quarantined VF pools as a bitmap. .Sh HARDWARE The .Nm diff --git a/sys/dev/ixgbe/if_ix.c b/sys/dev/ixgbe/if_ix.c index 28e79fa34e65..519c82304f54 100644 --- a/sys/dev/ixgbe/if_ix.c +++ b/sys/dev/ixgbe/if_ix.c @@ -3565,6 +3565,14 @@ ixgbe_add_device_sysctls(if_ctx_t ctx) "iov_dma_abort_flr_failures", CTLFLAG_RD, &sc->iov_dma_abort_flr_failures, 0, "Failed VF reset attempts after invalid-DMA events"); + SYSCTL_ADD_U64(ctx_list, child, OID_AUTO, + "iov_dma_abort_quarantines", CTLFLAG_RD, + &sc->iov_dma_abort_quarantines, 0, + "VFs quarantined after repeated invalid-DMA events"); + SYSCTL_ADD_U64(ctx_list, child, OID_AUTO, + "iov_quarantined_vfs", CTLFLAG_RD, + &sc->iov_quarantined_vfs, 0, + "Bitmap of quarantined VF pools"); } sc->enable_aim = ixgbe_enable_aim; diff --git a/sys/dev/ixgbe/if_sriov.c b/sys/dev/ixgbe/if_sriov.c index 6568ea9e7c18..32b4935e3b44 100644 --- a/sys/dev/ixgbe/if_sriov.c +++ b/sys/dev/ixgbe/if_sriov.c @@ -42,6 +42,7 @@ MALLOC_DEFINE(M_IXGBE_SRIOV, "ix_sriov", "ix SR-IOV allocations"); #define IXGBE_VF_MBX_CLEANUP_GRACE (2 * SBT_1S) +#define IXGBE_PRIMARY_ABORT_LIMIT 5 static const struct timeval ixgbe_mdd_log_interval = { 2, 0 }; static const struct timeval ixgbe_dma_abort_log_interval = { 10, 0 }; @@ -729,7 +730,7 @@ ixgbe_iov_recovery_task(void *context, int pending __unused) struct ixgbe_softc *sc; struct ixgbe_vf *vf; struct sx *ctx_lock; - bool new_event, report, scan, success; + bool new_event, quarantined, report, scan, success; u16 command, status; u32 tx_good; u64 recovery_vfs, stalled_vfs; @@ -759,6 +760,7 @@ ixgbe_iov_recovery_task(void *context, int pending __unused) stalled_vfs = 0; recovery_vf = -1; new_event = false; + quarantined = false; for (i = 0; i < num_vfs; i++) { if (sc->vfs[i].flags & IXGBE_VF_DMA_ABORT_PENDING) recovery_vfs |= 1ULL << i; @@ -804,6 +806,14 @@ ixgbe_iov_recovery_task(void *context, int pending __unused) ixgbe_vf_tx_sample_reset(vf); vf->flags |= IXGBE_VF_DMA_ABORT_PENDING; vf->flags &= ~IXGBE_VF_CTS; + vf->primary_abort_count++; + if (vf->primary_abort_count == + IXGBE_PRIMARY_ABORT_LIMIT) { + vf->flags |= IXGBE_VF_QUARANTINED; + sc->iov_dma_abort_quarantines++; + sc->iov_quarantined_vfs |= 1ULL << i; + quarantined = true; + } ixgbe_vf_enable_transmit(sc, vf); ixgbe_vf_enable_receive(sc, vf); IXGBE_WRITE_FLUSH(&sc->hw); @@ -844,11 +854,30 @@ ixgbe_iov_recovery_task(void *context, int pending __unused) success = pcie_flr(vfdev[i], flr_delay, true); } else success = false; - if (success) { + if (success && !(vf->flags & IXGBE_VF_QUARANTINED)) { /* Restore and verify the complete state for a usable VF. */ pci_restore_state(vfdev[i]); command = pci_read_config(vfdev[i], PCIR_COMMAND, 2); success = command == vf->pci_saved_command; + } else if (success) { + /* + * Leave the function in post-FLR configuration. Ensure that + * decode and bus mastering remain disabled, then refresh the + * PCI layer's cached Command state so a later restore cannot + * re-enable them. + */ + command = pci_read_config(vfdev[i], PCIR_COMMAND, 2); + if (command != UINT16_MAX) { + command &= ~(PCIM_CMD_PORTEN | PCIM_CMD_MEMEN | + PCIM_CMD_BUSMASTEREN); + pci_write_config(vfdev[i], PCIR_COMMAND, command, 2); + command = pci_read_config(vfdev[i], PCIR_COMMAND, 2); + } + success = command != UINT16_MAX && + (command & (PCIM_CMD_PORTEN | PCIM_CMD_MEMEN | + PCIM_CMD_BUSMASTEREN)) == 0; + if (success) + pci_save_state(vfdev[i]); } if (success) { vf->flags &= ~(IXGBE_VF_DMA_ABORT_PENDING | @@ -856,17 +885,24 @@ ixgbe_iov_recovery_task(void *context, int pending __unused) vf->pci_saved_command = 0; } else sc->iov_dma_abort_flr_failures++; - report = ratecheck(&vf->last_dma_abort_log, - &ixgbe_dma_abort_log_interval) != 0; - if (report && (new_event || !success)) { - if (success) - device_printf(sc->dev, - "invalid DMA target from VF %u; reset VF\n", - vf->pool); - else - device_printf(sc->dev, - "could not reset VF %u after an invalid DMA " - "target; VF remains disabled\n", vf->pool); + if (quarantined) { + device_printf(sc->dev, + "quarantined VF %u after %u invalid DMA targets%s\n", + vf->pool, IXGBE_PRIMARY_ABORT_LIMIT, + success ? "" : "; function-level reset failed"); + } else { + report = ratecheck(&vf->last_dma_abort_log, + &ixgbe_dma_abort_log_interval) != 0; + if (report && (new_event || !success)) { + if (success) + device_printf(sc->dev, + "invalid DMA target from VF %u; reset VF\n", + vf->pool); + else + device_printf(sc->dev, + "could not reset VF %u after an invalid DMA " + "target; VF remains disabled\n", vf->pool); + } } sx_xunlock(ctx_lock); for (i = 0; i < num_vfs; i++) { @@ -1703,6 +1739,7 @@ ixgbe_if_iov_init(if_ctx_t ctx, u16 num_vfs, const nvlist_t *config) sc->iov_pf_mdd_reset_pending = false; sc->iov_recovery_time = 0; sc->iov_recovery_cursor = 0; + sc->iov_quarantined_vfs = 0; ixgbe_init_mbx_params_pf(&sc->hw); sc->feat_en |= IXGBE_FEATURE_SRIOV; @@ -1720,6 +1757,7 @@ err_init_iov: sc->iov_mode = IXGBE_NO_VM; sc->iov_mbx_cleanup_pending = false; sc->iov_pf_mdd_reset_pending = false; + sc->iov_quarantined_vfs = 0; return (retval); } /* ixgbe_if_iov_init */ @@ -1792,6 +1830,7 @@ ixgbe_if_iov_uninit(if_ctx_t ctx) free(sc->vfs, M_IXGBE_SRIOV); sc->vfs = NULL; sc->feat_en &= ~IXGBE_FEATURE_SRIOV; + sc->iov_quarantined_vfs = 0; sc->pool = 0; sc->iov_mode = IXGBE_NO_VM; ixgbe_align_all_queue_indices(sc); @@ -1811,7 +1850,7 @@ ixgbe_init_vf(struct ixgbe_softc *sc, struct ixgbe_vf *vf) s32 error; hw = &sc->hw; - /* Discard stale recovery state after PF initialization. */ + /* Preserve quarantine until the SR-IOV configuration is destroyed. */ vf->flags &= ~(IXGBE_VF_INIT_DONE | IXGBE_VF_DMA_ABORT_PENDING | IXGBE_VF_PCI_STATE_SAVED | IXGBE_VF_MDD_BLOCKED | IXGBE_VF_MDD_NOTIFY_PENDING); @@ -1820,6 +1859,11 @@ ixgbe_init_vf(struct ixgbe_softc *sc, struct ixgbe_vf *vf) if (!(vf->flags & IXGBE_VF_ACTIVE)) return (IXGBE_SUCCESS); + if (vf->flags & IXGBE_VF_QUARANTINED) { + vf->flags &= ~IXGBE_VF_CTS; + return (IXGBE_SUCCESS); + } + vf_index = IXGBE_VF_INDEX(vf->pool); pfmbimr = IXGBE_READ_REG(hw, IXGBE_PFMBIMR(vf_index)); pfmbimr |= IXGBE_VF_BIT(vf->pool); diff --git a/sys/dev/ixgbe/ixgbe.h b/sys/dev/ixgbe/ixgbe.h index 1460e99889c2..90ce3e470686 100644 --- a/sys/dev/ixgbe/ixgbe.h +++ b/sys/dev/ixgbe/ixgbe.h @@ -370,6 +370,7 @@ struct ixgbe_vf { uint16_t pci_saved_command; uint32_t recovery_tx_head[IXGBE_VF_MAX_TX_QUEUES]; uint8_t xcast_mode; + uint8_t primary_abort_count; uint8_t recovery_tx_pending; sbintime_t mbx_cleanup_deadline; }; @@ -476,6 +477,8 @@ struct ixgbe_softc { uint8_t iov_recovery_cursor; uint64_t iov_dma_abort_events; uint64_t iov_dma_abort_flr_failures; + uint64_t iov_dma_abort_quarantines; + uint64_t iov_quarantined_vfs; /* Bypass */ struct ixgbe_bp_data bypass; diff --git a/sys/dev/ixgbe/ixgbe_sriov.h b/sys/dev/ixgbe/ixgbe_sriov.h index 55236f559eb0..dbe56ec1171c 100644 --- a/sys/dev/ixgbe/ixgbe_sriov.h +++ b/sys/dev/ixgbe/ixgbe_sriov.h @@ -53,9 +53,11 @@ #define IXGBE_VF_INIT_DONE (1U << 7) /* Hardware state is ready. */ #define IXGBE_VF_DMA_ABORT_PENDING (1U << 8) /* VF awaits a forced FLR. */ #define IXGBE_VF_PCI_STATE_SAVED (1U << 9) /* Preserve state for retry. */ -#define IXGBE_VF_IO_DISABLED IXGBE_VF_DMA_ABORT_PENDING +#define IXGBE_VF_QUARANTINED (1U << 10) /* VF exceeded fault limit. */ #define IXGBE_VF_MDD_BLOCKED (1U << 11) /* VF is gated after MDD. */ #define IXGBE_VF_MDD_NOTIFY_PENDING (1U << 12) /* Retry reset notice. */ +#define IXGBE_VF_IO_DISABLED \ + (IXGBE_VF_DMA_ABORT_PENDING | IXGBE_VF_QUARANTINED) #define IXGBE_VF_TRAFFIC_DISABLED \ (IXGBE_VF_IO_DISABLED | IXGBE_VF_MDD_BLOCKED) #define IXGBE_VF_INDEX(vmdq) ((vmdq) / 32)home | help
Want to link to this message? Use this
URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?6a75e1c4.41234.42acfc7f>