From owner-svn-src-all@FreeBSD.ORG Sun Mar 1 00:39:49 2015 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id 62E9731F; Sun, 1 Mar 2015 00:39:49 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 4E29F151; Sun, 1 Mar 2015 00:39:49 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.9/8.14.9) with ESMTP id t210dn54090089; Sun, 1 Mar 2015 00:39:49 GMT (envelope-from rstone@FreeBSD.org) Received: (from rstone@localhost) by svn.freebsd.org (8.14.9/8.14.9/Submit) id t210dm23090086; Sun, 1 Mar 2015 00:39:48 GMT (envelope-from rstone@FreeBSD.org) Message-Id: <201503010039.t210dm23090086@svn.freebsd.org> X-Authentication-Warning: svn.freebsd.org: rstone set sender to rstone@FreeBSD.org using -f From: Ryan Stone Date: Sun, 1 Mar 2015 00:39:48 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r279444 - in head/sys/amd64/vmm: . io X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.18-1 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 01 Mar 2015 00:39:49 -0000 Author: rstone Date: Sun Mar 1 00:39:48 2015 New Revision: 279444 URL: https://svnweb.freebsd.org/changeset/base/279444 Log: Allow passthrough devices to be hinted. Allow the ppt driver to attach to devices that were hinted to be passthrough devices by the PCI code creating them with a driver name of "ppt". Add a tunable that allows the IOMMU to be forced to be used. With SR-IOV passthrough devices the VFs may be created after vmm.ko is loaded. The current code will not initialize the IOMMU in that case, meaning that the passthrough devices can't actually be used. Differential Revision: https://reviews.freebsd.org/D73 Reviewed by: neel MFC after: 1 month Sponsored by: Sandvine Inc. Modified: head/sys/amd64/vmm/io/ppt.c head/sys/amd64/vmm/vmm.c Modified: head/sys/amd64/vmm/io/ppt.c ============================================================================== --- head/sys/amd64/vmm/io/ppt.c Sun Mar 1 00:39:40 2015 (r279443) +++ head/sys/amd64/vmm/io/ppt.c Sun Mar 1 00:39:48 2015 (r279444) @@ -56,7 +56,6 @@ __FBSDID("$FreeBSD$"); /* XXX locking */ -#define MAX_PPTDEVS (sizeof(pptdevs) / sizeof(pptdevs[0])) #define MAX_MSIMSGS 32 /* @@ -77,9 +76,10 @@ struct pptintr_arg { /* pptintr(pptin uint64_t msg_data; }; -static struct pptdev { +struct pptdev { device_t dev; struct vm *vm; /* owner of this device */ + TAILQ_ENTRY(pptdev) next; struct vm_memory_segment mmio[MAX_MMIOSEGS]; struct { int num_msgs; /* guest state */ @@ -99,7 +99,7 @@ static struct pptdev { void **cookie; struct pptintr_arg *arg; } msix; -} pptdevs[64]; +}; SYSCTL_DECL(_hw_vmm); SYSCTL_NODE(_hw_vmm, OID_AUTO, ppt, CTLFLAG_RW, 0, "bhyve passthru devices"); @@ -108,6 +108,8 @@ static int num_pptdevs; SYSCTL_INT(_hw_vmm_ppt, OID_AUTO, devices, CTLFLAG_RD, &num_pptdevs, 0, "number of pci passthru devices"); +static TAILQ_HEAD(, pptdev) pptdev_list = TAILQ_HEAD_INITIALIZER(pptdev_list); + static int ppt_probe(device_t dev) { @@ -125,26 +127,30 @@ ppt_probe(device_t dev) * - be allowed by administrator to be used in this role * - be an endpoint device */ - if (vmm_is_pptdev(bus, slot, func) && - (dinfo->cfg.hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_NORMAL) + if ((dinfo->cfg.hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_NORMAL) + return (ENXIO); + else if (vmm_is_pptdev(bus, slot, func)) return (0); else - return (ENXIO); + /* + * Returning BUS_PROBE_NOWILDCARD here matches devices that the + * SR-IOV infrastructure specified as "ppt" passthrough devices. + * All normal devices that did not have "ppt" specified as their + * driver will not be matched by this. + */ + return (BUS_PROBE_NOWILDCARD); } static int ppt_attach(device_t dev) { - int n; + struct pptdev *ppt; - if (num_pptdevs >= MAX_PPTDEVS) { - printf("ppt_attach: maximum number of pci passthrough devices " - "exceeded\n"); - return (ENXIO); - } + ppt = device_get_softc(dev); - n = num_pptdevs++; - pptdevs[n].dev = dev; + num_pptdevs++; + TAILQ_INSERT_TAIL(&pptdev_list, ppt, next); + ppt->dev = dev; if (bootverbose) device_printf(dev, "attached\n"); @@ -155,10 +161,14 @@ ppt_attach(device_t dev) static int ppt_detach(device_t dev) { - /* - * XXX check whether there are any pci passthrough devices assigned - * to guests before we allow this driver to detach. - */ + struct pptdev *ppt; + + ppt = device_get_softc(dev); + + if (ppt->vm != NULL) + return (EBUSY); + num_pptdevs--; + TAILQ_REMOVE(&pptdev_list, ppt, next); return (0); } @@ -172,22 +182,23 @@ static device_method_t ppt_methods[] = { }; static devclass_t ppt_devclass; -DEFINE_CLASS_0(ppt, ppt_driver, ppt_methods, 0); +DEFINE_CLASS_0(ppt, ppt_driver, ppt_methods, sizeof(struct pptdev)); DRIVER_MODULE(ppt, pci, ppt_driver, ppt_devclass, NULL, NULL); static struct pptdev * ppt_find(int bus, int slot, int func) { device_t dev; - int i, b, s, f; + struct pptdev *ppt; + int b, s, f; - for (i = 0; i < num_pptdevs; i++) { - dev = pptdevs[i].dev; + TAILQ_FOREACH(ppt, &pptdev_list, next) { + dev = ppt->dev; b = pci_get_bus(dev); s = pci_get_slot(dev); f = pci_get_function(dev); if (bus == b && slot == s && func == f) - return (&pptdevs[i]); + return (ppt); } return (NULL); } @@ -297,11 +308,12 @@ ppt_avail_devices(void) int ppt_assigned_devices(struct vm *vm) { - int i, num; + struct pptdev *ppt; + int num; num = 0; - for (i = 0; i < num_pptdevs; i++) { - if (pptdevs[i].vm == vm) + TAILQ_FOREACH(ppt, &pptdev_list, next) { + if (ppt->vm == vm) num++; } return (num); @@ -310,12 +322,11 @@ ppt_assigned_devices(struct vm *vm) boolean_t ppt_is_mmio(struct vm *vm, vm_paddr_t gpa) { - int i, n; + int i; struct pptdev *ppt; struct vm_memory_segment *seg; - for (n = 0; n < num_pptdevs; n++) { - ppt = &pptdevs[n]; + TAILQ_FOREACH(ppt, &pptdev_list, next) { if (ppt->vm != vm) continue; @@ -377,12 +388,13 @@ ppt_unassign_device(struct vm *vm, int b int ppt_unassign_all(struct vm *vm) { - int i, bus, slot, func; + struct pptdev *ppt; + int bus, slot, func; device_t dev; - for (i = 0; i < num_pptdevs; i++) { - if (pptdevs[i].vm == vm) { - dev = pptdevs[i].dev; + TAILQ_FOREACH(ppt, &pptdev_list, next) { + if (ppt->vm == vm) { + dev = ppt->dev; bus = pci_get_bus(dev); slot = pci_get_slot(dev); func = pci_get_function(dev); Modified: head/sys/amd64/vmm/vmm.c ============================================================================== --- head/sys/amd64/vmm/vmm.c Sun Mar 1 00:39:40 2015 (r279443) +++ head/sys/amd64/vmm/vmm.c Sun Mar 1 00:39:48 2015 (r279444) @@ -217,6 +217,11 @@ SYSCTL_INT(_hw_vmm, OID_AUTO, trace_gues &trace_guest_exceptions, 0, "Trap into hypervisor on all guest exceptions and reflect them back"); +static int vmm_force_iommu = 0; +TUNABLE_INT("hw.vmm.force_iommu", &vmm_force_iommu); +SYSCTL_INT(_hw_vmm, OID_AUTO, force_iommu, CTLFLAG_RDTUN, &vmm_force_iommu, 0, + "Force use of I/O MMU even if no passthrough devices were found."); + static void vcpu_cleanup(struct vm *vm, int i, bool destroy) { @@ -321,7 +326,7 @@ vmm_handler(module_t mod, int what, void switch (what) { case MOD_LOAD: vmmdev_init(); - if (ppt_avail_devices() > 0) + if (vmm_force_iommu || ppt_avail_devices() > 0) iommu_init(); error = vmm_init(); if (error == 0)