Date: Wed, 21 Aug 2019 22:18:01 +0000 (UTC) From: Warner Losh <imp@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r351356 - in head/sys: conf dev/ahci dev/nvme modules/nvme Message-ID: <201908212218.x7LMI1kO004971@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: imp Date: Wed Aug 21 22:18:01 2019 New Revision: 351356 URL: https://svnweb.freebsd.org/changeset/base/351356 Log: Create a AHCI attachment for nvme. Intel has created RST and many laptops from vendors like Lenovo and Asus. It's a mechanism for creating multiple boot devices under windows. It effectively hides the nvme drive inside of the ahci controller. The details are supposed to be a trade secret. However, there's a reverse engineered Linux driver, and this implements similar operations to allow nvme drives to attach. The ahci driver attaches nvme children that proxy the remapped resources to the child. nvme_ahci is just like nvme_pci, except it doesn't do the PCI specific things. That's moved into ahci where appropriate. When the nvme drive is remapped, MSI-x interrupts aren't forwarded (the linux driver doesn't know how to use this either). INTx interrupts are used instead. This is suboptimal, but usually sufficient for the laptops these parts are in. This is based loosely on https://www.spinics.net/lists/linux-ide/msg53364.html submitted, but not accepted by, Linux. It was written by Dan Williams. These changes were written from scratch by Olivier Houchard. Submitted by: cognet@ (Olivier Houchard) Added: head/sys/dev/nvme/nvme_ahci.c (contents, props changed) Modified: head/sys/conf/files head/sys/dev/ahci/ahci.c head/sys/dev/ahci/ahci.h head/sys/dev/ahci/ahci_pci.c head/sys/modules/nvme/Makefile Modified: head/sys/conf/files ============================================================================== --- head/sys/conf/files Wed Aug 21 22:17:55 2019 (r351355) +++ head/sys/conf/files Wed Aug 21 22:18:01 2019 (r351356) @@ -2479,6 +2479,7 @@ dev/nmdm/nmdm.c optional nmdm dev/null/null.c standard dev/nvd/nvd.c optional nvd nvme dev/nvme/nvme.c optional nvme +dev/nvme/nvme_ahci.c optional nvme ahci dev/nvme/nvme_ctrlr.c optional nvme dev/nvme/nvme_ctrlr_cmd.c optional nvme dev/nvme/nvme_ns.c optional nvme Modified: head/sys/dev/ahci/ahci.c ============================================================================== --- head/sys/dev/ahci/ahci.c Wed Aug 21 22:17:55 2019 (r351355) +++ head/sys/dev/ahci/ahci.c Wed Aug 21 22:18:01 2019 (r351356) @@ -347,6 +347,16 @@ ahci_attach(device_t dev) if ((ctlr->ichannels & (1 << unit)) == 0) device_disable(child); } + /* Attach any remapped NVME device */ + for (; unit < ctlr->channels + ctlr->remapped_devices; unit++) { + child = device_add_child(dev, "nvme", -1); + if (child == NULL) { + device_printf(dev, "failed to add remapped NVMe device"); + continue; + } + device_set_ivars(child, (void *)(intptr_t)(unit | AHCI_REMAPPED_UNIT)); + } + if (ctlr->caps & AHCI_CAP_EMS) { child = device_add_child(dev, "ahciem", -1); if (child == NULL) @@ -497,6 +507,12 @@ ahci_intr(void *data) ctlr->interrupt[unit].function(arg); } } + for (; unit < ctlr->channels + ctlr->remapped_devices; unit++) { + if ((arg = ctlr->interrupt[unit].argument)) { + ctlr->interrupt[unit].function(arg); + } + } + /* AHCI declares level triggered IS. */ if (!(ctlr->quirks & AHCI_Q_EDGEIS)) ATA_OUTL(ctlr->r_mem, AHCI_IS, is); @@ -546,12 +562,23 @@ ahci_alloc_resource(device_t dev, device_t child, int struct resource *res; rman_res_t st; int offset, size, unit; + bool is_remapped; unit = (intptr_t)device_get_ivars(child); + if (unit & AHCI_REMAPPED_UNIT) { + unit &= ~AHCI_REMAPPED_UNIT; + unit -= ctlr->channels; + is_remapped = true; + } else + is_remapped = false; res = NULL; switch (type) { case SYS_RES_MEMORY: - if (unit >= 0) { + if (is_remapped) { + offset = ctlr->remap_offset + unit * ctlr->remap_size; + size = ctlr->remap_size; + } + else if (unit >= 0) { offset = AHCI_OFFSET + (unit << 7); size = 128; } else if (*rid == 0) { @@ -612,7 +639,7 @@ ahci_setup_intr(device_t dev, device_t child, struct r void *argument, void **cookiep) { struct ahci_controller *ctlr = device_get_softc(dev); - int unit = (intptr_t)device_get_ivars(child); + int unit = (intptr_t)device_get_ivars(child) & ~AHCI_REMAPPED_UNIT; if (filter != NULL) { printf("ahci.c: we cannot use a filter here\n"); @@ -628,7 +655,7 @@ ahci_teardown_intr(device_t dev, device_t child, struc void *cookie) { struct ahci_controller *ctlr = device_get_softc(dev); - int unit = (intptr_t)device_get_ivars(child); + int unit = (intptr_t)device_get_ivars(child) & ~AHCI_REMAPPED_UNIT; ctlr->interrupt[unit].function = NULL; ctlr->interrupt[unit].argument = NULL; @@ -641,7 +668,7 @@ ahci_print_child(device_t dev, device_t child) int retval, channel; retval = bus_print_child_header(dev, child); - channel = (int)(intptr_t)device_get_ivars(child); + channel = (int)(intptr_t)device_get_ivars(child) & ~AHCI_REMAPPED_UNIT; if (channel >= 0) retval += printf(" at channel %d", channel); retval += bus_print_child_footer(dev, child); @@ -654,7 +681,7 @@ ahci_child_location_str(device_t dev, device_t child, { int channel; - channel = (int)(intptr_t)device_get_ivars(child); + channel = (int)(intptr_t)device_get_ivars(child) & ~AHCI_REMAPPED_UNIT; if (channel >= 0) snprintf(buf, buflen, "channel=%d", channel); return (0); Modified: head/sys/dev/ahci/ahci.h ============================================================================== --- head/sys/dev/ahci/ahci.h Wed Aug 21 22:17:55 2019 (r351355) +++ head/sys/dev/ahci/ahci.h Wed Aug 21 22:18:01 2019 (r351356) @@ -214,6 +214,7 @@ #define AHCI_CAP2_SADM 0x00000010 #define AHCI_CAP2_DESO 0x00000020 +#define AHCI_VSCAP 0xa4 #define AHCI_OFFSET 0x100 #define AHCI_STEP 0x80 @@ -318,6 +319,10 @@ /* Total main work area. */ #define AHCI_WORK_SIZE (AHCI_CT_OFFSET + AHCI_CT_SIZE * ch->numslots) + +/* NVMe remapped device */ +#define AHCI_REMAPPED_UNIT (1 << 31) + struct ahci_dma_prd { u_int64_t dba; u_int32_t reserved; @@ -518,6 +523,9 @@ struct ahci_controller { int cccv; /* CCC vector */ int direct; /* Direct command completion */ int msi; /* MSI interupts */ + int remapped_devices; /* Remapped NVMe devices */ + uint32_t remap_offset; + uint32_t remap_size; struct { void (*function)(void *); void *argument; Modified: head/sys/dev/ahci/ahci_pci.c ============================================================================== --- head/sys/dev/ahci/ahci_pci.c Wed Aug 21 22:17:55 2019 (r351355) +++ head/sys/dev/ahci/ahci_pci.c Wed Aug 21 22:18:01 2019 (r351356) @@ -495,6 +495,48 @@ ahci_pci_attach(device_t dev) &ctlr->r_rid, RF_ACTIVE))) return ENXIO; + /* + * Intel RAID hardware can remap NVMe devices inside its BAR. + * Try to detect this. Either we have to add the device + * here, or the user has to change the mode in the BIOS + * from RST to AHCI. + */ + if (pci_get_vendor(dev) == 0x8086) { + uint32_t vscap; + + vscap = ATA_INL(ctlr->r_mem, AHCI_VSCAP); + if (vscap & 1) { + uint32_t cap = ATA_INL(ctlr->r_mem, 0x800); /* Intel's REMAP CAP */ + int i; + + ctlr->remap_offset = 0x4000; + ctlr->remap_size = 0x4000; + + /* + * Check each of the devices that might be remapped to + * make sure they are an nvme device. At the present, + * nvme are the only known devices remapped. + */ + for (i = 0; i < 3; i++) { + if (cap & (1 << i) && + (ATA_INL(ctlr->r_mem, 0x880 + i * 0x80) == + ((PCIC_STORAGE << 16) | + (PCIS_STORAGE_NVM << 8) | + PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0))) { + ctlr->remapped_devices++; + } + } + + /* If we have any remapped device, disable MSI */ + if (ctlr->remapped_devices > 0) { + device_printf(dev, "Detected %d nvme remapped devices\n", + ctlr->remapped_devices); + ctlr->quirks |= (AHCI_Q_NOMSIX | AHCI_Q_NOMSI); + } + } + } + + if (ctlr->quirks & AHCI_Q_NOMSIX) msix_count = 0; Added: head/sys/dev/nvme/nvme_ahci.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/dev/nvme/nvme_ahci.c Wed Aug 21 22:18:01 2019 (r351356) @@ -0,0 +1,127 @@ +/*- + * Copyright (C) 2017 Olivier Houchard + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/buf.h> +#include <sys/bus.h> +#include <sys/conf.h> +#include <sys/proc.h> +#include <sys/smp.h> + +#include "nvme_private.h" + +static int nvme_ahci_probe(device_t dev); +static int nvme_ahci_attach(device_t dev); +static int nvme_ahci_detach(device_t dev); + +static device_method_t nvme_ahci_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, nvme_ahci_probe), + DEVMETHOD(device_attach, nvme_ahci_attach), + DEVMETHOD(device_detach, nvme_ahci_detach), + DEVMETHOD(device_shutdown, nvme_shutdown), + { 0, 0 } +}; + +static driver_t nvme_ahci_driver = { + "nvme", + nvme_ahci_methods, + sizeof(struct nvme_controller), +}; + +DRIVER_MODULE(nvme, ahci, nvme_ahci_driver, nvme_devclass, NULL, 0); +MODULE_VERSION(nvme_ahci, 1); + +static int +nvme_ahci_probe (device_t device) +{ + return (0); +} + +static int +nvme_ahci_attach(device_t dev) +{ + struct nvme_controller*ctrlr = DEVICE2SOFTC(dev); + int ret; + + /* Map MMIO registers */ + ctrlr->resource_id = 0; + + ctrlr->resource = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &ctrlr->resource_id, RF_ACTIVE); + + if(ctrlr->resource == NULL) { + nvme_printf(ctrlr, "unable to allocate mem resource\n"); + ret = ENOMEM; + goto bad; + } + ctrlr->bus_tag = rman_get_bustag(ctrlr->resource); + ctrlr->bus_handle = rman_get_bushandle(ctrlr->resource); + ctrlr->regs = (struct nvme_registers *)ctrlr->bus_handle; + + /* Allocate and setup IRQ */ + ctrlr->rid = 0; + ctrlr->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, + &ctrlr->rid, RF_SHAREABLE | RF_ACTIVE); + + if (ctrlr->res == NULL) { + nvme_printf(ctrlr, "unable to allocate shared IRQ\n"); + ret = ENOMEM; + goto bad; + } + + ctrlr->msix_enabled = 0; + ctrlr->num_io_queues = 1; + ctrlr->num_cpus_per_ioq = mp_ncpus; + if (bus_setup_intr(dev, ctrlr->res, + INTR_TYPE_MISC | INTR_MPSAFE, NULL, nvme_ctrlr_intx_handler, + ctrlr, &ctrlr->tag) != 0) { + nvme_printf(ctrlr, "unable to setup intx handler\n"); + ret = ENOMEM; + goto bad; + } + ctrlr->tag = (void *)0x1; + + return nvme_attach(dev); +bad: + if (ctrlr->resource != NULL) { + bus_release_resource(dev, SYS_RES_MEMORY, + ctrlr->resource_id, ctrlr->resource); + } + if (ctrlr->res) + bus_release_resource(ctrlr->dev, SYS_RES_IRQ, + rman_get_rid(ctrlr->res), ctrlr->res); + return (ret); +} + +static int +nvme_ahci_detach(device_t dev) +{ + + return (nvme_detach(dev)); +} Modified: head/sys/modules/nvme/Makefile ============================================================================== --- head/sys/modules/nvme/Makefile Wed Aug 21 22:17:55 2019 (r351355) +++ head/sys/modules/nvme/Makefile Wed Aug 21 22:18:01 2019 (r351356) @@ -5,6 +5,7 @@ KMOD = nvme SRCS = nvme.c \ + nvme_ahci.c \ nvme_ctrlr.c \ nvme_ctrlr_cmd.c \ nvme_ns.c \
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201908212218.x7LMI1kO004971>