Date: Mon, 14 Dec 2020 20:40:21 +0000 (UTC) From: John Baldwin <jhb@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org Subject: svn commit: r368641 - in stable/12: lib/libvmmapi sys/amd64/include sys/amd64/vmm sys/amd64/vmm/io usr.sbin/bhyve Message-ID: <202012142040.0BEKeLET096525@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jhb Date: Mon Dec 14 20:40:21 2020 New Revision: 368641 URL: https://svnweb.freebsd.org/changeset/base/368641 Log: MFC 368003: Honor the disabled setting for MSI-X interrupts for passthrough devices. Add a new ioctl to disable all MSI-X interrupts for a PCI passthrough device and invoke it if a write to the MSI-X capability registers disables MSI-X. This avoids leaving MSI-X interrupts enabled on the host if a guest device driver has disabled them (e.g. as part of detaching a guest device driver). This was found by Chelsio QA when testing that a Linux guest could switch from MSI-X to MSI interrupts when using the cxgb4vf driver. While here, explicitly fail requests to enable MSI on a passthrough device if MSI-X is enabled and vice versa. Modified: stable/12/lib/libvmmapi/vmmapi.c stable/12/lib/libvmmapi/vmmapi.h stable/12/sys/amd64/include/vmm_dev.h stable/12/sys/amd64/vmm/io/ppt.c stable/12/sys/amd64/vmm/io/ppt.h stable/12/sys/amd64/vmm/vmm_dev.c stable/12/usr.sbin/bhyve/pci_passthru.c Directory Properties: stable/12/ (props changed) Modified: stable/12/lib/libvmmapi/vmmapi.c ============================================================================== --- stable/12/lib/libvmmapi/vmmapi.c Mon Dec 14 19:47:08 2020 (r368640) +++ stable/12/lib/libvmmapi/vmmapi.c Mon Dec 14 20:40:21 2020 (r368641) @@ -952,6 +952,19 @@ vm_setup_pptdev_msix(struct vmctx *ctx, int vcpu, int return ioctl(ctx->fd, VM_PPTDEV_MSIX, &pptmsix); } +int +vm_disable_pptdev_msix(struct vmctx *ctx, int bus, int slot, int func) +{ + struct vm_pptdev ppt; + + bzero(&ppt, sizeof(ppt)); + ppt.bus = bus; + ppt.slot = slot; + ppt.func = func; + + return ioctl(ctx->fd, VM_PPTDEV_DISABLE_MSIX, &ppt); +} + uint64_t * vm_get_stats(struct vmctx *ctx, int vcpu, struct timeval *ret_tv, int *ret_entries) @@ -1552,7 +1565,8 @@ vm_get_ioctls(size_t *len) VM_ISA_DEASSERT_IRQ, VM_ISA_PULSE_IRQ, VM_ISA_SET_IRQ_TRIGGER, VM_SET_CAPABILITY, VM_GET_CAPABILITY, VM_BIND_PPTDEV, VM_UNBIND_PPTDEV, VM_MAP_PPTDEV_MMIO, VM_PPTDEV_MSI, - VM_PPTDEV_MSIX, VM_INJECT_NMI, VM_STATS, VM_STAT_DESC, + VM_PPTDEV_MSIX, VM_PPTDEV_DISABLE_MSIX, + VM_INJECT_NMI, VM_STATS, VM_STAT_DESC, VM_SET_X2APIC_STATE, VM_GET_X2APIC_STATE, VM_GET_HPET_CAPABILITIES, VM_GET_GPA_PMAP, VM_GLA2GPA, VM_GLA2GPA_NOFAULT, Modified: stable/12/lib/libvmmapi/vmmapi.h ============================================================================== --- stable/12/lib/libvmmapi/vmmapi.h Mon Dec 14 19:47:08 2020 (r368640) +++ stable/12/lib/libvmmapi/vmmapi.h Mon Dec 14 20:40:21 2020 (r368641) @@ -168,6 +168,7 @@ int vm_setup_pptdev_msi(struct vmctx *ctx, int vcpu, i int vm_setup_pptdev_msix(struct vmctx *ctx, int vcpu, int bus, int slot, int func, int idx, uint64_t addr, uint64_t msg, uint32_t vector_control); +int vm_disable_pptdev_msix(struct vmctx *ctx, int bus, int slot, int func); int vm_get_intinfo(struct vmctx *ctx, int vcpu, uint64_t *i1, uint64_t *i2); int vm_set_intinfo(struct vmctx *ctx, int vcpu, uint64_t exit_intinfo); Modified: stable/12/sys/amd64/include/vmm_dev.h ============================================================================== --- stable/12/sys/amd64/include/vmm_dev.h Mon Dec 14 19:47:08 2020 (r368640) +++ stable/12/sys/amd64/include/vmm_dev.h Mon Dec 14 20:40:21 2020 (r368641) @@ -281,6 +281,7 @@ enum { IOCNUM_MAP_PPTDEV_MMIO = 42, IOCNUM_PPTDEV_MSI = 43, IOCNUM_PPTDEV_MSIX = 44, + IOCNUM_PPTDEV_DISABLE_MSIX = 45, /* statistics */ IOCNUM_VM_STATS = 50, @@ -378,6 +379,8 @@ enum { _IOW('v', IOCNUM_PPTDEV_MSI, struct vm_pptdev_msi) #define VM_PPTDEV_MSIX \ _IOW('v', IOCNUM_PPTDEV_MSIX, struct vm_pptdev_msix) +#define VM_PPTDEV_DISABLE_MSIX \ + _IOW('v', IOCNUM_PPTDEV_DISABLE_MSIX, struct vm_pptdev) #define VM_INJECT_NMI \ _IOW('v', IOCNUM_INJECT_NMI, struct vm_nmi) #define VM_STATS \ Modified: stable/12/sys/amd64/vmm/io/ppt.c ============================================================================== --- stable/12/sys/amd64/vmm/io/ppt.c Mon Dec 14 19:47:08 2020 (r368640) +++ stable/12/sys/amd64/vmm/io/ppt.c Mon Dec 14 20:40:21 2020 (r368641) @@ -517,6 +517,10 @@ ppt_setup_msi(struct vm *vm, int vcpu, int bus, int sl if (ppt->vm != vm) /* Make sure we own this device */ return (EBUSY); + /* Reject attempts to enable MSI while MSI-X is active. */ + if (ppt->msix.num_msgs != 0 && numvec != 0) + return (EBUSY); + /* Free any allocated resources */ ppt_teardown_msi(ppt); @@ -606,6 +610,10 @@ ppt_setup_msix(struct vm *vm, int vcpu, int bus, int s if (ppt->vm != vm) /* Make sure we own this device */ return (EBUSY); + /* Reject attempts to enable MSI-X while MSI is active. */ + if (ppt->msi.num_msgs != 0) + return (EBUSY); + dinfo = device_get_ivars(ppt->dev); if (!dinfo) return (ENXIO); @@ -697,5 +705,20 @@ ppt_setup_msix(struct vm *vm, int vcpu, int bus, int s ppt_teardown_msix_intr(ppt, idx); } + return (0); +} + +int +ppt_disable_msix(struct vm *vm, int bus, int slot, int func) +{ + struct pptdev *ppt; + + ppt = ppt_find(bus, slot, func); + if (ppt == NULL) + return (ENOENT); + if (ppt->vm != vm) /* Make sure we own this device */ + return (EBUSY); + + ppt_teardown_msix(ppt); return (0); } Modified: stable/12/sys/amd64/vmm/io/ppt.h ============================================================================== --- stable/12/sys/amd64/vmm/io/ppt.h Mon Dec 14 19:47:08 2020 (r368640) +++ stable/12/sys/amd64/vmm/io/ppt.h Mon Dec 14 20:40:21 2020 (r368641) @@ -38,6 +38,7 @@ int ppt_setup_msi(struct vm *vm, int vcpu, int bus, in uint64_t addr, uint64_t msg, int numvec); int ppt_setup_msix(struct vm *vm, int vcpu, int bus, int slot, int func, int idx, uint64_t addr, uint64_t msg, uint32_t vector_control); +int ppt_disable_msix(struct vm *vm, int bus, int slot, int func); int ppt_assigned_devices(struct vm *vm); bool ppt_is_mmio(struct vm *vm, vm_paddr_t gpa); Modified: stable/12/sys/amd64/vmm/vmm_dev.c ============================================================================== --- stable/12/sys/amd64/vmm/vmm_dev.c Mon Dec 14 19:47:08 2020 (r368640) +++ stable/12/sys/amd64/vmm/vmm_dev.c Mon Dec 14 20:40:21 2020 (r368641) @@ -496,6 +496,11 @@ vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t da pptmsix->addr, pptmsix->msg, pptmsix->vector_control); break; + case VM_PPTDEV_DISABLE_MSIX: + pptdev = (struct vm_pptdev *)data; + error = ppt_disable_msix(sc->vm, pptdev->bus, pptdev->slot, + pptdev->func); + break; case VM_MAP_PPTDEV_MMIO: pptmmio = (struct vm_pptdev_mmio *)data; error = ppt_map_mmio(sc->vm, pptmmio->bus, pptmmio->slot, Modified: stable/12/usr.sbin/bhyve/pci_passthru.c ============================================================================== --- stable/12/usr.sbin/bhyve/pci_passthru.c Mon Dec 14 19:47:08 2020 (r368640) +++ stable/12/usr.sbin/bhyve/pci_passthru.c Mon Dec 14 20:40:21 2020 (r368641) @@ -872,6 +872,11 @@ passthru_cfgwrite(struct vmctx *ctx, int vcpu, struct if (error) err(1, "vm_setup_pptdev_msix"); } + } else { + error = vm_disable_pptdev_msix(ctx, sc->psc_sel.pc_bus, + sc->psc_sel.pc_dev, sc->psc_sel.pc_func); + if (error) + err(1, "vm_disable_pptdev_msix"); } return (0); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202012142040.0BEKeLET096525>