Date: Fri, 5 Apr 2019 19:25:26 +0000 (UTC) From: Konstantin Belousov <kib@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r345963 - in head/sys: amd64/vmm/io dev/pci Message-ID: <201904051925.x35JPRQM085944@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kib Date: Fri Apr 5 19:25:26 2019 New Revision: 345963 URL: https://svnweb.freebsd.org/changeset/base/345963 Log: Implement resets for PCI buses and PCIe bridges. For PCI device (i.e. child of a PCI bus), reset tries FLR if implemented and worked, and falls to power reset otherwise. For PCIe bus (child of a PCIe bridge or root port), reset disables PCIe link and then re-trains it, performing what is known as link-level reset. Reviewed by: imp (previous version), jhb (previous version) Sponsored by: Mellanox Technologies MFC after: 2 weeks Differential revision: https://reviews.freebsd.org/D19646 Modified: head/sys/amd64/vmm/io/ppt.c head/sys/dev/pci/pci.c head/sys/dev/pci/pci_pci.c head/sys/dev/pci/pcivar.h Modified: head/sys/amd64/vmm/io/ppt.c ============================================================================== --- head/sys/amd64/vmm/io/ppt.c Fri Apr 5 18:37:48 2019 (r345962) +++ head/sys/amd64/vmm/io/ppt.c Fri Apr 5 19:25:26 2019 (r345963) @@ -356,25 +356,12 @@ ppt_is_mmio(struct vm *vm, vm_paddr_t gpa) static void ppt_pci_reset(device_t dev) { - int ps; if (pcie_flr(dev, - max(pcie_get_max_completion_timeout(dev) / 1000, 10), - true)) + max(pcie_get_max_completion_timeout(dev) / 1000, 10), true)) return; - /* - * If FLR fails, attempt a power-management reset by cycling - * the device in/out of D3 state. - * PCI spec says we can only go into D3 state from D0 state. - * Transition from D[12] into D0 before going to D3 state. - */ - ps = pci_get_powerstate(dev); - if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3) - pci_set_powerstate(dev, PCI_POWERSTATE_D0); - if (pci_get_powerstate(dev) != PCI_POWERSTATE_D3) - pci_set_powerstate(dev, PCI_POWERSTATE_D3); - pci_set_powerstate(dev, ps); + pci_power_reset(dev); } int Modified: head/sys/dev/pci/pci.c ============================================================================== --- head/sys/dev/pci/pci.c Fri Apr 5 18:37:48 2019 (r345962) +++ head/sys/dev/pci/pci.c Fri Apr 5 19:25:26 2019 (r345963) @@ -126,6 +126,10 @@ static int pci_remap_intr_method(device_t bus, device u_int irq); static void pci_hint_device_unit(device_t acdev, device_t child, const char *name, int *unitp); +static int pci_reset_post(device_t dev, device_t child); +static int pci_reset_prepare(device_t dev, device_t child); +static int pci_reset_child(device_t dev, device_t child, + int flags); static int pci_get_id_method(device_t dev, device_t child, enum pci_id_type type, uintptr_t *rid); @@ -150,6 +154,9 @@ static device_method_t pci_methods[] = { DEVMETHOD(bus_driver_added, pci_driver_added), DEVMETHOD(bus_setup_intr, pci_setup_intr), DEVMETHOD(bus_teardown_intr, pci_teardown_intr), + DEVMETHOD(bus_reset_prepare, pci_reset_prepare), + DEVMETHOD(bus_reset_post, pci_reset_post), + DEVMETHOD(bus_reset_child, pci_reset_child), DEVMETHOD(bus_get_dma_tag, pci_get_dma_tag), DEVMETHOD(bus_get_resource_list,pci_get_resource_list), @@ -6385,6 +6392,89 @@ pcie_flr(device_t dev, u_int max_delay, bool force) PCIEM_STA_TRANSACTION_PND) pci_printf(&dinfo->cfg, "Transactions pending after FLR!\n"); return (true); +} + +int +pci_power_reset(device_t dev) +{ + int ps; + + ps = pci_get_powerstate(dev); + if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3) + pci_set_powerstate(dev, PCI_POWERSTATE_D0); + pci_set_powerstate(dev, PCI_POWERSTATE_D3); + pci_set_powerstate(dev, ps); + return (0); +} + +/* + * Try link drop and retrain of the downstream port of upstream + * switch, for PCIe. According to the PCIe 3.0 spec 6.6.1, this must + * cause Conventional Hot reset of the device in the slot. + * Alternative, for PCIe, could be the secondary bus reset initiatied + * on the upstream switch PCIR_BRIDGECTL_1, bit 6. + */ +int +pcie_link_reset(device_t port, int pcie_location) +{ + uint16_t v; + + v = pci_read_config(port, pcie_location + PCIER_LINK_CTL, 2); + v |= PCIEM_LINK_CTL_LINK_DIS; + pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2); + pause_sbt("pcier1", mstosbt(20), 0, 0); + v &= ~PCIEM_LINK_CTL_LINK_DIS; + v |= PCIEM_LINK_CTL_RETRAIN_LINK; + pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2); + pause_sbt("pcier2", mstosbt(100), 0, 0); /* 100 ms */ + v = pci_read_config(port, pcie_location + PCIER_LINK_STA, 2); + return ((v & PCIEM_LINK_STA_TRAINING) != 0 ? ETIMEDOUT : 0); +} + +static int +pci_reset_post(device_t dev, device_t child) +{ + + if (dev == device_get_parent(child)) + pci_restore_state(child); + return (0); +} + +static int +pci_reset_prepare(device_t dev, device_t child) +{ + + if (dev == device_get_parent(child)) + pci_save_state(child); + return (0); +} + +static int +pci_reset_child(device_t dev, device_t child, int flags) +{ + int error; + + if (dev == NULL || device_get_parent(child) != dev) + return (0); + if ((flags & DEVF_RESET_DETACH) != 0) { + error = device_get_state(child) == DS_ATTACHED ? + device_detach(child) : 0; + } else { + error = BUS_SUSPEND_CHILD(dev, child); + } + if (error == 0) { + if (!pcie_flr(child, 1000, false)) { + error = BUS_RESET_PREPARE(dev, child); + if (error == 0) + pci_power_reset(child); + BUS_RESET_POST(dev, child); + } + if ((flags & DEVF_RESET_DETACH) != 0) + device_probe_and_attach(child); + else + BUS_RESUME_CHILD(dev, child); + } + return (error); } const struct pci_device_table * Modified: head/sys/dev/pci/pci_pci.c ============================================================================== --- head/sys/dev/pci/pci_pci.c Fri Apr 5 18:37:48 2019 (r345962) +++ head/sys/dev/pci/pci_pci.c Fri Apr 5 19:25:26 2019 (r345963) @@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/malloc.h> #include <sys/module.h> +#include <sys/pciio.h> #include <sys/rman.h> #include <sys/sysctl.h> #include <sys/systm.h> @@ -80,6 +81,7 @@ static void pcib_pcie_dll_timeout(void *arg); #endif static int pcib_request_feature_default(device_t pcib, device_t dev, enum pci_feature feature); +static int pcib_reset_child(device_t dev, device_t child, int flags); static device_method_t pcib_methods[] = { /* Device interface */ @@ -106,6 +108,7 @@ static device_method_t pcib_methods[] = { DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), + DEVMETHOD(bus_reset_child, pcib_reset_child), /* pcib interface */ DEVMETHOD(pcib_maxslots, pcib_ari_maxslots), @@ -2908,4 +2911,32 @@ pcib_request_feature_default(device_t pcib, device_t d */ bus = device_get_parent(pcib); return (PCIB_REQUEST_FEATURE(device_get_parent(bus), dev, feature)); +} + +static int +pcib_reset_child(device_t dev, device_t child, int flags) +{ + struct pci_devinfo *pdinfo; + int error; + + error = 0; + if (dev == NULL || device_get_parent(child) != dev) + goto out; + error = ENXIO; + if (device_get_devclass(child) != devclass_find("pci")) + goto out; + pdinfo = device_get_ivars(dev); + if (pdinfo->cfg.pcie.pcie_location != 0 && + (pdinfo->cfg.pcie.pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT || + pdinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT)) { + error = bus_helper_reset_prepare(child, flags); + if (error == 0) { + error = pcie_link_reset(dev, + pdinfo->cfg.pcie.pcie_location); + /* XXXKIB call _post even if error != 0 ? */ + bus_helper_reset_post(child, flags); + } + } +out: + return (error); } Modified: head/sys/dev/pci/pcivar.h ============================================================================== --- head/sys/dev/pci/pcivar.h Fri Apr 5 18:37:48 2019 (r345962) +++ head/sys/dev/pci/pcivar.h Fri Apr 5 19:25:26 2019 (r345963) @@ -681,6 +681,7 @@ int pci_get_max_read_req(device_t dev); void pci_restore_state(device_t dev); void pci_save_state(device_t dev); int pci_set_max_read_req(device_t dev, int size); +int pci_power_reset(device_t dev); uint32_t pcie_read_config(device_t dev, int reg, int width); void pcie_write_config(device_t dev, int reg, uint32_t value, int width); uint32_t pcie_adjust_config(device_t dev, int reg, uint32_t mask, @@ -688,6 +689,7 @@ uint32_t pcie_adjust_config(device_t dev, int reg, uin bool pcie_flr(device_t dev, u_int max_delay, bool force); int pcie_get_max_completion_timeout(device_t dev); bool pcie_wait_for_pending_transactions(device_t dev, u_int max_delay); +int pcie_link_reset(device_t port, int pcie_location); void pci_print_faulted_dev(void);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201904051925.x35JPRQM085944>