Date: Wed, 3 Jan 2018 19:24:57 +0000 (UTC) From: Navdeep Parhar <np@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r327528 - in head: share/man/man4 sys/dev/cxgbe Message-ID: <201801031924.w03JOvM9029277@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: np Date: Wed Jan 3 19:24:57 2018 New Revision: 327528 URL: https://svnweb.freebsd.org/changeset/base/327528 Log: cxgbe(4): Add a knob to enable/disable PCIe relaxed ordering. Disable it by default when running on Intel CPUs. This is a crude fix for the performance issues alluded to in these Linux commits: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=87e09cdec4dae08acdb4aa49beb793c19d73e73e https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=a99b646afa8a02571ea298bedca6592d818229cd MFC after: 1 week Sponsored by: Chelsio Communications Modified: head/share/man/man4/cxgbe.4 head/sys/dev/cxgbe/t4_main.c Modified: head/share/man/man4/cxgbe.4 ============================================================================== --- head/share/man/man4/cxgbe.4 Wed Jan 3 19:24:21 2018 (r327527) +++ head/share/man/man4/cxgbe.4 Wed Jan 3 19:24:57 2018 (r327528) @@ -243,6 +243,13 @@ Permitted interrupt types. Bit 0 represents INTx (line interrupts), bit 1 MSI, and bit 2 MSI-X. The default is 7 (all allowed). The driver selects the best possible type out of the allowed types. +.It Va hw.cxgbe.pcie_relaxed_ordering +PCIe Relaxed Ordering. +-1 indicates the driver should determine whether to enable or disable PCIe RO. +0 disables PCIe RO. +1 enables PCIe RO. +2 indicates the driver should not modify the PCIe RO setting. +The default is -1. .It Va hw.cxgbe.fw_install 0 prohibits the driver from installing a firmware on the card. 1 allows the driver to install a new firmware if internal driver Modified: head/sys/dev/cxgbe/t4_main.c ============================================================================== --- head/sys/dev/cxgbe/t4_main.c Wed Jan 3 19:24:21 2018 (r327527) +++ head/sys/dev/cxgbe/t4_main.c Wed Jan 3 19:24:57 2018 (r327528) @@ -63,6 +63,8 @@ __FBSDID("$FreeBSD$"); #include <net/rss_config.h> #endif #if defined(__i386__) || defined(__amd64__) +#include <machine/md_var.h> +#include <machine/cputypes.h> #include <vm/vm.h> #include <vm/pmap.h> #endif @@ -454,7 +456,17 @@ TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine) static int t4_num_vis = 1; TUNABLE_INT("hw.cxgbe.num_vis", &t4_num_vis); +/* + * PCIe Relaxed Ordering. + * -1: driver should figure out a good value. + * 0: disable RO. + * 1: enable RO. + * 2: leave RO alone. + */ +static int pcie_relaxed_ordering = -1; +TUNABLE_INT("hw.cxgbe.pcie_relaxed_ordering", &pcie_relaxed_ordering); + /* Functions used by VIs to obtain unique MAC addresses for each VI. */ static int vi_mac_funcs[] = { FW_VI_FUNC_ETH, @@ -856,10 +868,16 @@ t4_attach(device_t dev) pci_set_max_read_req(dev, 4096); v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); - v |= PCIEM_CTL_RELAXED_ORD_ENABLE; - pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); - sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); + if (pcie_relaxed_ordering == 0 && + (v | PCIEM_CTL_RELAXED_ORD_ENABLE) != 0) { + v &= ~PCIEM_CTL_RELAXED_ORD_ENABLE; + pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); + } else if (pcie_relaxed_ordering == 1 && + (v & PCIEM_CTL_RELAXED_ORD_ENABLE) == 0) { + v |= PCIEM_CTL_RELAXED_ORD_ENABLE; + pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); + } } sc->sge_gts_reg = MYPF_REG(A_SGE_PF_GTS); @@ -9961,6 +9979,14 @@ tweak_tunables(void) if (t4_num_vis > nitems(vi_mac_funcs)) { t4_num_vis = nitems(vi_mac_funcs); printf("cxgbe: number of VIs limited to %d\n", t4_num_vis); + } + + if (pcie_relaxed_ordering < 0 || pcie_relaxed_ordering > 2) { + pcie_relaxed_ordering = 1; +#if defined(__i386__) || defined(__amd64__) + if (cpu_vendor_id == CPU_VENDOR_INTEL) + pcie_relaxed_ordering = 0; +#endif } }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201801031924.w03JOvM9029277>