From owner-svn-src-stable@freebsd.org Wed Apr 27 23:04:43 2016 Return-Path: Delivered-To: svn-src-stable@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 89ED1B1FAD5; Wed, 27 Apr 2016 23:04:43 +0000 (UTC) (envelope-from jhb@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 615DE12C8; Wed, 27 Apr 2016 23:04:43 +0000 (UTC) (envelope-from jhb@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id u3RN4glh076022; Wed, 27 Apr 2016 23:04:42 GMT (envelope-from jhb@FreeBSD.org) Received: (from jhb@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id u3RN4guA076020; Wed, 27 Apr 2016 23:04:42 GMT (envelope-from jhb@FreeBSD.org) Message-Id: <201604272304.u3RN4guA076020@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: jhb set sender to jhb@FreeBSD.org using -f From: John Baldwin Date: Wed, 27 Apr 2016 23:04:42 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r298724 - stable/10/usr.sbin/bhyve X-SVN-Group: stable-10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-stable@freebsd.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: SVN commit messages for all the -stable branches of the src tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 27 Apr 2016 23:04:43 -0000 Author: jhb Date: Wed Apr 27 23:04:42 2016 New Revision: 298724 URL: https://svnweb.freebsd.org/changeset/base/298724 Log: MFC 297932,298295: Improvements for PCI passthru devices. 297932: Handle PBA that shares a page with MSI-X table for passthrough devices. If the PBA shares a page with the MSI-X table, map the shared page via /dev/mem and emulate accesses to the portion of the PBA in the shared page by accessing the mapped page. 298295: Always emit an error message on passthru configuration errors. Previously, many errors (such as the PCI device not being attached to the ppt(4) driver) resulted in bhyve silently exiting without starting the virtual machine. Now any errors encountered when configuring a virtual slot for a PCI passthru device should be noted on stderr. Modified: stable/10/usr.sbin/bhyve/pci_emul.h stable/10/usr.sbin/bhyve/pci_passthru.c Directory Properties: stable/10/ (props changed) Modified: stable/10/usr.sbin/bhyve/pci_emul.h ============================================================================== --- stable/10/usr.sbin/bhyve/pci_emul.h Wed Apr 27 21:51:24 2016 (r298723) +++ stable/10/usr.sbin/bhyve/pci_emul.h Wed Apr 27 23:04:42 2016 (r298724) @@ -142,6 +142,8 @@ struct pci_devinst { int pba_size; int function_mask; struct msix_table_entry *table; /* allocated at runtime */ + void *pba_page; + int pba_page_offset; } pi_msix; void *pi_arg; /* devemu-private data */ Modified: stable/10/usr.sbin/bhyve/pci_passthru.c ============================================================================== --- stable/10/usr.sbin/bhyve/pci_passthru.c Wed Apr 27 21:51:24 2016 (r298723) +++ stable/10/usr.sbin/bhyve/pci_passthru.c Wed Apr 27 23:04:42 2016 (r298724) @@ -31,6 +31,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include @@ -42,7 +43,7 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include +#include #include #include @@ -59,6 +60,10 @@ __FBSDID("$FreeBSD$"); #define _PATH_DEVIO "/dev/io" #endif +#ifndef _PATH_MEM +#define _PATH_MEM "/dev/mem" +#endif + #define LEGACY_SUPPORT 1 #define MSIX_TABLE_COUNT(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1) @@ -66,6 +71,7 @@ __FBSDID("$FreeBSD$"); static int pcifd = -1; static int iofd = -1; +static int memfd = -1; struct passthru_softc { struct pci_devinst *psc_pi; @@ -279,6 +285,35 @@ msix_table_read(struct passthru_softc *s int index; pi = sc->psc_pi; + if (offset >= pi->pi_msix.pba_offset && + offset < pi->pi_msix.pba_offset + pi->pi_msix.pba_size) { + switch(size) { + case 1: + src8 = (uint8_t *)(pi->pi_msix.pba_page + offset - + pi->pi_msix.pba_page_offset); + data = *src8; + break; + case 2: + src16 = (uint16_t *)(pi->pi_msix.pba_page + offset - + pi->pi_msix.pba_page_offset); + data = *src16; + break; + case 4: + src32 = (uint32_t *)(pi->pi_msix.pba_page + offset - + pi->pi_msix.pba_page_offset); + data = *src32; + break; + case 8: + src64 = (uint64_t *)(pi->pi_msix.pba_page + offset - + pi->pi_msix.pba_page_offset); + data = *src64; + break; + default: + return (-1); + } + return (data); + } + if (offset < pi->pi_msix.table_offset) return (-1); @@ -320,12 +355,44 @@ msix_table_write(struct vmctx *ctx, int { struct pci_devinst *pi; struct msix_table_entry *entry; - uint32_t *dest; + uint8_t *dest8; + uint16_t *dest16; + uint32_t *dest32; + uint64_t *dest64; size_t entry_offset; uint32_t vector_control; int error, index; pi = sc->psc_pi; + if (offset >= pi->pi_msix.pba_offset && + offset < pi->pi_msix.pba_offset + pi->pi_msix.pba_size) { + switch(size) { + case 1: + dest8 = (uint8_t *)(pi->pi_msix.pba_page + offset - + pi->pi_msix.pba_page_offset); + *dest8 = data; + break; + case 2: + dest16 = (uint16_t *)(pi->pi_msix.pba_page + offset - + pi->pi_msix.pba_page_offset); + *dest16 = data; + break; + case 4: + dest32 = (uint32_t *)(pi->pi_msix.pba_page + offset - + pi->pi_msix.pba_page_offset); + *dest32 = data; + break; + case 8: + dest64 = (uint64_t *)(pi->pi_msix.pba_page + offset - + pi->pi_msix.pba_page_offset); + *dest64 = data; + break; + default: + break; + } + return; + } + if (offset < pi->pi_msix.table_offset) return; @@ -342,8 +409,8 @@ msix_table_write(struct vmctx *ctx, int assert(entry_offset % 4 == 0); vector_control = entry->vector_control; - dest = (uint32_t *)((void *)entry + entry_offset); - *dest = data; + dest32 = (uint32_t *)((void *)entry + entry_offset); + *dest32 = data; /* If MSI-X hasn't been enabled, do nothing */ if (pi->pi_msix.enabled) { /* If the entry is masked, don't set it up */ @@ -386,28 +453,44 @@ init_msix_table(struct vmctx *ctx, struc table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE; table_size = roundup2(table_size, 4096); + idx = pi->pi_msix.table_bar; + start = pi->pi_bar[idx].addr; + remaining = pi->pi_bar[idx].size; + if (pi->pi_msix.pba_bar == pi->pi_msix.table_bar) { pba_offset = pi->pi_msix.pba_offset; pba_size = pi->pi_msix.pba_size; if (pba_offset >= table_offset + table_size || table_offset >= pba_offset + pba_size) { /* - * The PBA can reside in the same BAR as the MSI-x - * tables as long as it does not overlap with any - * naturally aligned page occupied by the tables. + * If the PBA does not share a page with the MSI-x + * tables, no PBA emulation is required. */ + pi->pi_msix.pba_page = NULL; + pi->pi_msix.pba_page_offset = 0; } else { - /* Need to also emulate the PBA, not supported yet */ - printf("Unsupported MSI-X configuration: %d/%d/%d\n", - b, s, f); - return (-1); + /* + * The PBA overlaps with either the first or last + * page of the MSI-X table region. Map the + * appropriate page. + */ + if (pba_offset <= table_offset) + pi->pi_msix.pba_page_offset = table_offset; + else + pi->pi_msix.pba_page_offset = table_offset + + table_size - 4096; + pi->pi_msix.pba_page = mmap(NULL, 4096, PROT_READ | + PROT_WRITE, MAP_SHARED, memfd, start + + pi->pi_msix.pba_page_offset); + if (pi->pi_msix.pba_page == MAP_FAILED) { + warn( + "Failed to map PBA page for MSI-X on %d/%d/%d", + b, s, f); + return (-1); + } } } - idx = pi->pi_msix.table_bar; - start = pi->pi_bar[idx].addr; - remaining = pi->pi_bar[idx].size; - /* Map everything before the MSI-X table */ if (table_offset > 0) { len = table_offset; @@ -476,7 +559,7 @@ cfginitbar(struct vmctx *ctx, struct pas if (bartype != PCIBAR_IO) { if (((base | size) & PAGE_MASK) != 0) { - printf("passthru device %d/%d/%d BAR %d: " + warnx("passthru device %d/%d/%d BAR %d: " "base %#lx or size %#lx not page aligned\n", sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func, i, base, size); @@ -534,11 +617,17 @@ cfginit(struct vmctx *ctx, struct pci_de sc->psc_sel.pc_dev = slot; sc->psc_sel.pc_func = func; - if (cfginitmsi(sc) != 0) + if (cfginitmsi(sc) != 0) { + warnx("failed to initialize MSI for PCI %d/%d/%d", + bus, slot, func); goto done; + } - if (cfginitbar(ctx, sc) != 0) + if (cfginitbar(ctx, sc) != 0) { + warnx("failed to initialize BARs for PCI %d/%d/%d", + bus, slot, func); goto done; + } error = 0; /* success */ done: @@ -556,28 +645,45 @@ passthru_init(struct vmctx *ctx, struct memflags = vm_get_memflags(ctx); if (!(memflags & VM_MEM_F_WIRED)) { - fprintf(stderr, "passthru requires guest memory to be wired\n"); + warnx("passthru requires guest memory to be wired"); goto done; } if (pcifd < 0) { pcifd = open(_PATH_DEVPCI, O_RDWR, 0); - if (pcifd < 0) + if (pcifd < 0) { + warn("failed to open %s", _PATH_DEVPCI); goto done; + } } if (iofd < 0) { iofd = open(_PATH_DEVIO, O_RDWR, 0); - if (iofd < 0) + if (iofd < 0) { + warn("failed to open %s", _PATH_DEVIO); goto done; + } + } + + if (memfd < 0) { + memfd = open(_PATH_MEM, O_RDWR, 0); + if (memfd < 0) { + warn("failed to open %s", _PATH_MEM); + goto done; + } } if (opts == NULL || - sscanf(opts, "%d/%d/%d", &bus, &slot, &func) != 3) + sscanf(opts, "%d/%d/%d", &bus, &slot, &func) != 3) { + warnx("invalid passthru options"); goto done; + } - if (vm_assign_pptdev(ctx, bus, slot, func) != 0) + if (vm_assign_pptdev(ctx, bus, slot, func) != 0) { + warnx("PCI device at %d/%d/%d is not using the ppt(4) driver", + bus, slot, func); goto done; + } sc = calloc(1, sizeof(struct passthru_softc)); @@ -688,10 +794,8 @@ passthru_cfgwrite(struct vmctx *ctx, int sc->psc_sel.pc_dev, sc->psc_sel.pc_func, pi->pi_msi.addr, pi->pi_msi.msg_data, pi->pi_msi.maxmsgnum); - if (error != 0) { - printf("vm_setup_pptdev_msi error %d\r\n", errno); - exit(1); - } + if (error != 0) + err(1, "vm_setup_pptdev_msi"); return (0); } @@ -707,11 +811,8 @@ passthru_cfgwrite(struct vmctx *ctx, int pi->pi_msix.table[i].msg_data, pi->pi_msix.table[i].vector_control); - if (error) { - printf("vm_setup_pptdev_msix error " - "%d\r\n", errno); - exit(1); - } + if (error) + err(1, "vm_setup_pptdev_msix"); } } return (0);