Date: Tue, 9 Jul 2013 23:12:26 +0000 (UTC) From: Marius Strobl <marius@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r253120 - head/sys/dev/pci Message-ID: <201307092312.r69NCQY1082902@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: marius Date: Tue Jul 9 23:12:26 2013 New Revision: 253120 URL: http://svnweb.freebsd.org/changeset/base/253120 Log: - As it turns out, not only MSI-X is broken for devices passed through by VMware up to at least ESXi 5.1. Actually, using INTx in that case instead may still result in interrupt storms, with MSI being the only working option in some configurations. So introduce a PCI_QUIRK_DISABLE_MSIX quirk which only blacklists MSI-X but not also MSI and use it for the VMware PCI-PCI-bridges. Note that, currently, we still assume that if MSI doesn't work, MSI-X won't work either - but that's part of the internal logic and not guaranteed as part of the API contract. While at it, add and employ a pci_has_quirk() helper. Reported and tested by: Paul Bucher - Use NULL instead of 0 for pointers. Submitted by: jhb (mostly) Approved by: jhb MFC after: 3 days Modified: head/sys/dev/pci/pci.c head/sys/dev/pci/pci_pci.c head/sys/dev/pci/pcib_private.h head/sys/dev/pci/pcivar.h Modified: head/sys/dev/pci/pci.c ============================================================================== --- head/sys/dev/pci/pci.c Tue Jul 9 22:04:35 2013 (r253119) +++ head/sys/dev/pci/pci.c Tue Jul 9 23:12:26 2013 (r253120) @@ -78,6 +78,7 @@ __FBSDID("$FreeBSD$"); (((cfg)->hdrtype == PCIM_HDRTYPE_NORMAL && reg == PCIR_BIOS) || \ ((cfg)->hdrtype == PCIM_HDRTYPE_BRIDGE && reg == PCIR_BIOS_1)) +static int pci_has_quirk(uint32_t devid, int quirk); static pci_addr_t pci_mapbase(uint64_t mapreg); static const char *pci_maptype(uint64_t mapreg); static int pci_mapsize(uint64_t testval); @@ -119,6 +120,7 @@ static void pci_enable_msix(device_t de static void pci_mask_msix(device_t dev, u_int index); static void pci_unmask_msix(device_t dev, u_int index); static int pci_msi_blacklisted(void); +static int pci_msix_blacklisted(void); static void pci_resume_msi(device_t dev); static void pci_resume_msix(device_t dev); static int pci_remap_intr_method(device_t bus, device_t dev, @@ -185,7 +187,7 @@ static device_method_t pci_methods[] = { DEFINE_CLASS_0(pci, pci_driver, pci_methods, sizeof(struct pci_softc)); static devclass_t pci_devclass; -DRIVER_MODULE(pci, pcib, pci_driver, pci_devclass, pci_modevent, 0); +DRIVER_MODULE(pci, pcib, pci_driver, pci_devclass, pci_modevent, NULL); MODULE_VERSION(pci, 1); static char *pci_vendordata; @@ -195,15 +197,16 @@ struct pci_quirk { uint32_t devid; /* Vendor/device of the card */ int type; #define PCI_QUIRK_MAP_REG 1 /* PCI map register in weird place */ -#define PCI_QUIRK_DISABLE_MSI 2 /* MSI/MSI-X doesn't work */ +#define PCI_QUIRK_DISABLE_MSI 2 /* Neither MSI nor MSI-X work */ #define PCI_QUIRK_ENABLE_MSI_VM 3 /* Older chipset in VM where MSI works */ #define PCI_QUIRK_UNMAP_REG 4 /* Ignore PCI map register */ +#define PCI_QUIRK_DISABLE_MSIX 5 /* MSI-X doesn't work */ int arg1; int arg2; }; static const struct pci_quirk pci_quirks[] = { - /* The Intel 82371AB and 82443MX has a map register at offset 0x90. */ + /* The Intel 82371AB and 82443MX have a map register at offset 0x90. */ { 0x71138086, PCI_QUIRK_MAP_REG, 0x90, 0 }, { 0x719b8086, PCI_QUIRK_MAP_REG, 0x90, 0 }, /* As does the Serverworks OSB4 (the SMBus mapping register) */ @@ -238,8 +241,8 @@ static const struct pci_quirk pci_quirks * MSI-X allocation doesn't work properly for devices passed through * by VMware up to at least ESXi 5.1. */ - { 0x079015ad, PCI_QUIRK_DISABLE_MSI, 0, 0 }, /* PCI/PCI-X */ - { 0x07a015ad, PCI_QUIRK_DISABLE_MSI, 0, 0 }, /* PCIe */ + { 0x079015ad, PCI_QUIRK_DISABLE_MSIX, 0, 0 }, /* PCI/PCI-X */ + { 0x07a015ad, PCI_QUIRK_DISABLE_MSIX, 0, 0 }, /* PCIe */ /* * Some virtualization environments emulate an older chipset @@ -321,7 +324,7 @@ SYSCTL_INT(_hw_pci, OID_AUTO, enable_msi static int pci_honor_msi_blacklist = 1; TUNABLE_INT("hw.pci.honor_msi_blacklist", &pci_honor_msi_blacklist); SYSCTL_INT(_hw_pci, OID_AUTO, honor_msi_blacklist, CTLFLAG_RD, - &pci_honor_msi_blacklist, 1, "Honor chipset blacklist for MSI"); + &pci_honor_msi_blacklist, 1, "Honor chipset blacklist for MSI/MSI-X"); #if defined(__i386__) || defined(__amd64__) static int pci_usb_takeover = 1; @@ -334,6 +337,18 @@ SYSCTL_INT(_hw_pci, OID_AUTO, usb_early_ Disable this if you depend on BIOS emulation of USB devices, that is\n\ you use USB devices (like keyboard or mouse) but do not load USB drivers"); +static int +pci_has_quirk(uint32_t devid, int quirk) +{ + const struct pci_quirk *q; + + for (q = &pci_quirks[0]; q->devid; q++) { + if (q->devid == devid && q->type == quirk) + return (1); + } + return (0); +} + /* Find a device_t by bus/slot/function in domain 0 */ device_t @@ -1426,8 +1441,8 @@ pci_alloc_msix_method(device_t dev, devi if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0) return (ENXIO); - /* If MSI is blacklisted for this system, fail. */ - if (pci_msi_blacklisted()) + /* If MSI-X is blacklisted for this system, fail. */ + if (pci_msix_blacklisted()) return (ENXIO); /* MSI-X capability present? */ @@ -1983,38 +1998,15 @@ pci_remap_intr_method(device_t bus, devi int pci_msi_device_blacklisted(device_t dev) { - const struct pci_quirk *q; if (!pci_honor_msi_blacklist) return (0); - for (q = &pci_quirks[0]; q->devid; q++) { - if (q->devid == pci_get_devid(dev) && - q->type == PCI_QUIRK_DISABLE_MSI) - return (1); - } - return (0); -} - -/* - * Returns true if a specified chipset supports MSI when it is - * emulated hardware in a virtual machine. - */ -static int -pci_msi_vm_chipset(device_t dev) -{ - const struct pci_quirk *q; - - for (q = &pci_quirks[0]; q->devid; q++) { - if (q->devid == pci_get_devid(dev) && - q->type == PCI_QUIRK_ENABLE_MSI_VM) - return (1); - } - return (0); + return (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSI)); } /* - * Determine if MSI is blacklisted globally on this sytem. Currently, + * Determine if MSI is blacklisted globally on this system. Currently, * we just check for blacklisted chipsets as represented by the * host-PCI bridge at device 0:0:0. In the future, it may become * necessary to check other system attributes, such as the kenv values @@ -2031,9 +2023,14 @@ pci_msi_blacklisted(void) /* Blacklist all non-PCI-express and non-PCI-X chipsets. */ if (!(pcie_chipset || pcix_chipset)) { if (vm_guest != VM_GUEST_NO) { + /* + * Whitelist older chipsets in virtual + * machines known to support MSI. + */ dev = pci_find_bsf(0, 0, 0); if (dev != NULL) - return (pci_msi_vm_chipset(dev) == 0); + return (!pci_has_quirk(pci_get_devid(dev), + PCI_QUIRK_ENABLE_MSI_VM)); } return (1); } @@ -2045,6 +2042,45 @@ pci_msi_blacklisted(void) } /* + * Returns true if the specified device is blacklisted because MSI-X + * doesn't work. Note that this assumes that if MSI doesn't work, + * MSI-X doesn't either. + */ +int +pci_msix_device_blacklisted(device_t dev) +{ + + if (!pci_honor_msi_blacklist) + return (0); + + if (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSIX)) + return (1); + + return (pci_msi_device_blacklisted(dev)); +} + +/* + * Determine if MSI-X is blacklisted globally on this system. If MSI + * is blacklisted, assume that MSI-X is as well. Check for additional + * chipsets where MSI works but MSI-X does not. + */ +static int +pci_msix_blacklisted(void) +{ + device_t dev; + + if (!pci_honor_msi_blacklist) + return (0); + + dev = pci_find_bsf(0, 0, 0); + if (dev != NULL && pci_has_quirk(pci_get_devid(dev), + PCI_QUIRK_DISABLE_MSIX)) + return (1); + + return (pci_msi_blacklisted()); +} + +/* * Attempt to allocate *count MSI messages. The actual number allocated is * returned in *count. After this function returns, each message will be * available to the driver as SYS_RES_IRQ resources starting at a rid 1. Modified: head/sys/dev/pci/pci_pci.c ============================================================================== --- head/sys/dev/pci/pci_pci.c Tue Jul 9 22:04:35 2013 (r253119) +++ head/sys/dev/pci/pci_pci.c Tue Jul 9 23:12:26 2013 (r253120) @@ -100,7 +100,7 @@ static device_method_t pcib_methods[] = static devclass_t pcib_devclass; DEFINE_CLASS_0(pcib, pcib_driver, pcib_methods, sizeof(struct pcib_softc)); -DRIVER_MODULE(pcib, pci, pcib_driver, pcib_devclass, 0, 0); +DRIVER_MODULE(pcib, pci, pcib_driver, pcib_devclass, NULL, NULL); #ifdef NEW_PCIB /* @@ -624,6 +624,9 @@ pcib_attach_common(device_t dev) if (pci_msi_device_blacklisted(dev)) sc->flags |= PCIB_DISABLE_MSI; + if (pci_msix_device_blacklisted(dev)) + sc->flags |= PCIB_DISABLE_MSIX; + /* * Intel 815, 845 and other chipsets say they are PCI-PCI bridges, * but have a ProgIF of 0x80. The 82801 family (AA, AB, BAM/CAM, @@ -1379,7 +1382,7 @@ pcib_alloc_msix(device_t pcib, device_t struct pcib_softc *sc = device_get_softc(pcib); device_t bus; - if (sc->flags & PCIB_DISABLE_MSI) + if (sc->flags & PCIB_DISABLE_MSIX) return (ENXIO); bus = device_get_parent(pcib); return (PCIB_ALLOC_MSIX(device_get_parent(bus), dev, irq)); Modified: head/sys/dev/pci/pcib_private.h ============================================================================== --- head/sys/dev/pci/pcib_private.h Tue Jul 9 22:04:35 2013 (r253119) +++ head/sys/dev/pci/pcib_private.h Tue Jul 9 23:12:26 2013 (r253120) @@ -91,6 +91,7 @@ struct pcib_softc uint32_t flags; /* flags */ #define PCIB_SUBTRACTIVE 0x1 #define PCIB_DISABLE_MSI 0x2 +#define PCIB_DISABLE_MSIX 0x4 uint16_t command; /* command register */ u_int domain; /* domain number */ u_int pribus; /* primary bus number */ Modified: head/sys/dev/pci/pcivar.h ============================================================================== --- head/sys/dev/pci/pcivar.h Tue Jul 9 22:04:35 2013 (r253119) +++ head/sys/dev/pci/pcivar.h Tue Jul 9 23:12:26 2013 (r253120) @@ -490,6 +490,7 @@ device_t pci_find_class(uint8_t class, u int pci_pending_msix(device_t dev, u_int index); int pci_msi_device_blacklisted(device_t dev); +int pci_msix_device_blacklisted(device_t dev); void pci_ht_map_msi(device_t dev, uint64_t addr);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201307092312.r69NCQY1082902>