Skip site navigation (1)Skip section navigation (2)
Date:      Fri, 5 Apr 2019 19:25:26 +0000 (UTC)
From:      Konstantin Belousov <kib@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r345963 - in head/sys: amd64/vmm/io dev/pci
Message-ID:  <201904051925.x35JPRQM085944@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: kib
Date: Fri Apr  5 19:25:26 2019
New Revision: 345963
URL: https://svnweb.freebsd.org/changeset/base/345963

Log:
  Implement resets for PCI buses and PCIe bridges.
  
  For PCI device (i.e. child of a PCI bus), reset tries FLR if
  implemented and worked, and falls to power reset otherwise.
  
  For PCIe bus (child of a PCIe bridge or root port), reset
  disables PCIe link and then re-trains it, performing what is known as
  link-level reset.
  
  Reviewed by:	imp (previous version), jhb (previous version)
  Sponsored by:	Mellanox Technologies
  MFC after:	2 weeks
  Differential revision:	https://reviews.freebsd.org/D19646

Modified:
  head/sys/amd64/vmm/io/ppt.c
  head/sys/dev/pci/pci.c
  head/sys/dev/pci/pci_pci.c
  head/sys/dev/pci/pcivar.h

Modified: head/sys/amd64/vmm/io/ppt.c
==============================================================================
--- head/sys/amd64/vmm/io/ppt.c	Fri Apr  5 18:37:48 2019	(r345962)
+++ head/sys/amd64/vmm/io/ppt.c	Fri Apr  5 19:25:26 2019	(r345963)
@@ -356,25 +356,12 @@ ppt_is_mmio(struct vm *vm, vm_paddr_t gpa)
 static void
 ppt_pci_reset(device_t dev)
 {
-	int ps;
 
 	if (pcie_flr(dev,
-		     max(pcie_get_max_completion_timeout(dev) / 1000, 10),
-		     true))
+	     max(pcie_get_max_completion_timeout(dev) / 1000, 10), true))
 		return;
 
-	/*
-	 * If FLR fails, attempt a power-management reset by cycling
-	 * the device in/out of D3 state.
-	 * PCI spec says we can only go into D3 state from D0 state.
-	 * Transition from D[12] into D0 before going to D3 state.
-	 */
-	ps = pci_get_powerstate(dev);
-	if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
-		pci_set_powerstate(dev, PCI_POWERSTATE_D0);
-	if (pci_get_powerstate(dev) != PCI_POWERSTATE_D3)
-		pci_set_powerstate(dev, PCI_POWERSTATE_D3);
-	pci_set_powerstate(dev, ps);
+	pci_power_reset(dev);
 }
 
 int

Modified: head/sys/dev/pci/pci.c
==============================================================================
--- head/sys/dev/pci/pci.c	Fri Apr  5 18:37:48 2019	(r345962)
+++ head/sys/dev/pci/pci.c	Fri Apr  5 19:25:26 2019	(r345963)
@@ -126,6 +126,10 @@ static int		pci_remap_intr_method(device_t bus, device
 			    u_int irq);
 static void		pci_hint_device_unit(device_t acdev, device_t child,
 			    const char *name, int *unitp);
+static int		pci_reset_post(device_t dev, device_t child);
+static int		pci_reset_prepare(device_t dev, device_t child);
+static int		pci_reset_child(device_t dev, device_t child,
+			    int flags);
 
 static int		pci_get_id_method(device_t dev, device_t child,
 			    enum pci_id_type type, uintptr_t *rid);
@@ -150,6 +154,9 @@ static device_method_t pci_methods[] = {
 	DEVMETHOD(bus_driver_added,	pci_driver_added),
 	DEVMETHOD(bus_setup_intr,	pci_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	pci_teardown_intr),
+	DEVMETHOD(bus_reset_prepare,	pci_reset_prepare),
+	DEVMETHOD(bus_reset_post,	pci_reset_post),
+	DEVMETHOD(bus_reset_child,	pci_reset_child),
 
 	DEVMETHOD(bus_get_dma_tag,	pci_get_dma_tag),
 	DEVMETHOD(bus_get_resource_list,pci_get_resource_list),
@@ -6385,6 +6392,89 @@ pcie_flr(device_t dev, u_int max_delay, bool force)
 	    PCIEM_STA_TRANSACTION_PND)
 		pci_printf(&dinfo->cfg, "Transactions pending after FLR!\n");
 	return (true);
+}
+
+int
+pci_power_reset(device_t dev)
+{
+	int ps;
+
+	ps = pci_get_powerstate(dev);
+	if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
+		pci_set_powerstate(dev, PCI_POWERSTATE_D0);
+	pci_set_powerstate(dev, PCI_POWERSTATE_D3);
+	pci_set_powerstate(dev, ps);
+	return (0);
+}
+
+/*
+ * Try link drop and retrain of the downstream port of upstream
+ * switch, for PCIe.  According to the PCIe 3.0 spec 6.6.1, this must
+ * cause Conventional Hot reset of the device in the slot.
+ * Alternative, for PCIe, could be the secondary bus reset initiatied
+ * on the upstream switch PCIR_BRIDGECTL_1, bit 6.
+ */
+int
+pcie_link_reset(device_t port, int pcie_location)
+{
+	uint16_t v;
+
+	v = pci_read_config(port, pcie_location + PCIER_LINK_CTL, 2);
+	v |= PCIEM_LINK_CTL_LINK_DIS;
+	pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2);
+	pause_sbt("pcier1", mstosbt(20), 0, 0);
+	v &= ~PCIEM_LINK_CTL_LINK_DIS;
+	v |= PCIEM_LINK_CTL_RETRAIN_LINK;
+	pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2);
+	pause_sbt("pcier2", mstosbt(100), 0, 0); /* 100 ms */
+	v = pci_read_config(port, pcie_location + PCIER_LINK_STA, 2);
+	return ((v & PCIEM_LINK_STA_TRAINING) != 0 ? ETIMEDOUT : 0);
+}
+
+static int
+pci_reset_post(device_t dev, device_t child)
+{
+
+	if (dev == device_get_parent(child))
+		pci_restore_state(child);
+	return (0);
+}
+
+static int
+pci_reset_prepare(device_t dev, device_t child)
+{
+
+	if (dev == device_get_parent(child))
+		pci_save_state(child);
+	return (0);
+}
+
+static int
+pci_reset_child(device_t dev, device_t child, int flags)
+{
+	int error;
+
+	if (dev == NULL || device_get_parent(child) != dev)
+		return (0);
+	if ((flags & DEVF_RESET_DETACH) != 0) {
+		error = device_get_state(child) == DS_ATTACHED ?
+		    device_detach(child) : 0;
+	} else {
+		error = BUS_SUSPEND_CHILD(dev, child);
+	}
+	if (error == 0) {
+		if (!pcie_flr(child, 1000, false)) {
+			error = BUS_RESET_PREPARE(dev, child);
+			if (error == 0)
+				pci_power_reset(child);
+			BUS_RESET_POST(dev, child);
+		}
+		if ((flags & DEVF_RESET_DETACH) != 0)
+			device_probe_and_attach(child);
+		else
+			BUS_RESUME_CHILD(dev, child);
+	}
+	return (error);
 }
 
 const struct pci_device_table *

Modified: head/sys/dev/pci/pci_pci.c
==============================================================================
--- head/sys/dev/pci/pci_pci.c	Fri Apr  5 18:37:48 2019	(r345962)
+++ head/sys/dev/pci/pci_pci.c	Fri Apr  5 19:25:26 2019	(r345963)
@@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #include <sys/module.h>
+#include <sys/pciio.h>
 #include <sys/rman.h>
 #include <sys/sysctl.h>
 #include <sys/systm.h>
@@ -80,6 +81,7 @@ static void		pcib_pcie_dll_timeout(void *arg);
 #endif
 static int		pcib_request_feature_default(device_t pcib, device_t dev,
 			    enum pci_feature feature);
+static int		pcib_reset_child(device_t dev, device_t child, int flags);
 
 static device_method_t pcib_methods[] = {
     /* Device interface */
@@ -106,6 +108,7 @@ static device_method_t pcib_methods[] = {
     DEVMETHOD(bus_deactivate_resource,	bus_generic_deactivate_resource),
     DEVMETHOD(bus_setup_intr,		bus_generic_setup_intr),
     DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
+    DEVMETHOD(bus_reset_child,		pcib_reset_child),
 
     /* pcib interface */
     DEVMETHOD(pcib_maxslots,		pcib_ari_maxslots),
@@ -2908,4 +2911,32 @@ pcib_request_feature_default(device_t pcib, device_t d
 	 */
 	bus = device_get_parent(pcib);
 	return (PCIB_REQUEST_FEATURE(device_get_parent(bus), dev, feature));
+}
+
+static int
+pcib_reset_child(device_t dev, device_t child, int flags)
+{
+	struct pci_devinfo *pdinfo;
+	int error;
+
+	error = 0;
+	if (dev == NULL || device_get_parent(child) != dev)
+		goto out;
+	error = ENXIO;
+	if (device_get_devclass(child) != devclass_find("pci"))
+		goto out;
+	pdinfo = device_get_ivars(dev);
+	if (pdinfo->cfg.pcie.pcie_location != 0 &&
+	    (pdinfo->cfg.pcie.pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT ||
+	    pdinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT)) {
+		error = bus_helper_reset_prepare(child, flags);
+		if (error == 0) {
+			error = pcie_link_reset(dev,
+			    pdinfo->cfg.pcie.pcie_location);
+			/* XXXKIB call _post even if error != 0 ? */
+			bus_helper_reset_post(child, flags);
+		}
+	}
+out:
+	return (error);
 }

Modified: head/sys/dev/pci/pcivar.h
==============================================================================
--- head/sys/dev/pci/pcivar.h	Fri Apr  5 18:37:48 2019	(r345962)
+++ head/sys/dev/pci/pcivar.h	Fri Apr  5 19:25:26 2019	(r345963)
@@ -681,6 +681,7 @@ int	pci_get_max_read_req(device_t dev);
 void	pci_restore_state(device_t dev);
 void	pci_save_state(device_t dev);
 int	pci_set_max_read_req(device_t dev, int size);
+int	pci_power_reset(device_t dev);
 uint32_t pcie_read_config(device_t dev, int reg, int width);
 void	pcie_write_config(device_t dev, int reg, uint32_t value, int width);
 uint32_t pcie_adjust_config(device_t dev, int reg, uint32_t mask,
@@ -688,6 +689,7 @@ uint32_t pcie_adjust_config(device_t dev, int reg, uin
 bool	pcie_flr(device_t dev, u_int max_delay, bool force);
 int	pcie_get_max_completion_timeout(device_t dev);
 bool	pcie_wait_for_pending_transactions(device_t dev, u_int max_delay);
+int	pcie_link_reset(device_t port, int pcie_location);
 
 void	pci_print_faulted_dev(void);
 



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201904051925.x35JPRQM085944>