From owner-svn-src-all@freebsd.org Wed May 30 03:00:58 2018 Return-Path: Delivered-To: svn-src-all@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 960DAEF6DB8; Wed, 30 May 2018 03:00:58 +0000 (UTC) (envelope-from jhibbits@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 475BC6B1F9; Wed, 30 May 2018 03:00:58 +0000 (UTC) (envelope-from jhibbits@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 2880624D32; Wed, 30 May 2018 03:00:58 +0000 (UTC) (envelope-from jhibbits@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id w4U30wOL041303; Wed, 30 May 2018 03:00:58 GMT (envelope-from jhibbits@FreeBSD.org) Received: (from jhibbits@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id w4U30wlv041302; Wed, 30 May 2018 03:00:58 GMT (envelope-from jhibbits@FreeBSD.org) Message-Id: <201805300300.w4U30wlv041302@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: jhibbits set sender to jhibbits@FreeBSD.org using -f From: Justin Hibbits Date: Wed, 30 May 2018 03:00:58 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r334367 - head/sys/powerpc/powernv X-SVN-Group: head X-SVN-Commit-Author: jhibbits X-SVN-Commit-Paths: head/sys/powerpc/powernv X-SVN-Commit-Revision: 334367 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.26 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 30 May 2018 03:00:58 -0000 Author: jhibbits Date: Wed May 30 03:00:57 2018 New Revision: 334367 URL: https://svnweb.freebsd.org/changeset/base/334367 Log: Make opal_pci driver work with POWER9 Summary: Coupled with r334365, this makes PCI work on POWER9. There is still more to do to fully exploit the hardware capabilities, but this is sufficient to enable USB and ethernet controllers on a POWER9 Talos II system. Reviewed by: nwhitehorn, leitao Differential Revision: https://reviews.freebsd.org/D15566 Modified: head/sys/powerpc/powernv/opal_pci.c Modified: head/sys/powerpc/powernv/opal_pci.c ============================================================================== --- head/sys/powerpc/powernv/opal_pci.c Wed May 30 02:47:23 2018 (r334366) +++ head/sys/powerpc/powernv/opal_pci.c Wed May 30 03:00:57 2018 (r334367) @@ -60,7 +60,7 @@ __FBSDID("$FreeBSD$"); #include "opal.h" #define OPAL_PCI_TCE_MAX_ENTRIES (1024*1024UL) -#define OPAL_PCI_TCE_SEG_SIZE (16*1024*1024UL) +#define OPAL_PCI_TCE_DEFAULT_SEG_SIZE (16*1024*1024UL) #define OPAL_PCI_TCE_R (1UL << 0) #define OPAL_PCI_TCE_W (1UL << 1) #define PHB3_TCE_KILL_INVAL_ALL (1UL << 63) @@ -195,15 +195,52 @@ pci_phb3_tce_invalidate_entire(struct opalpci_softc *s mb(); } +/* Simple function to round to a power of 2 */ +static uint64_t +round_pow2(uint64_t val) +{ + + return (1 << (flsl(val + (val - 1)) - 1)); +} + +/* + * Starting with skiboot 5.10 PCIe nodes have a new property, + * "ibm,supported-tce-sizes", to denote the TCE sizes available. This allows us + * to avoid hard-coding the maximum TCE size allowed, and instead provide a sane + * default (however, the "sane" default, which works for all targets, is 64k, + * limiting us to 64GB if we have 1M entries. + */ +static uint64_t +max_tce_size(device_t dev) +{ + phandle_t node; + cell_t sizes[64]; /* Property is a list of bit-widths, up to 64-bits */ + int count; + + node = ofw_bus_get_node(dev); + + count = OF_getencprop(node, "ibm,supported-tce-sizes", + sizes, sizeof(sizes)); + if (count < sizeof(cell_t)) + return OPAL_PCI_TCE_DEFAULT_SEG_SIZE; + + count /= sizeof(cell_t); + + return (1ULL << sizes[count - 1]); +} + static int opalpci_attach(device_t dev) { struct opalpci_softc *sc; - cell_t id[2], m64window[6], npe; + cell_t id[2], m64ranges[2], m64window[6], npe; phandle_t node; int i, err; uint64_t maxmem; uint64_t entries; + uint64_t tce_size; + uint64_t tce_tbl_size; + int m64bar; int rid; sc = device_get_softc(dev); @@ -236,6 +273,7 @@ opalpci_attach(device_t dev) return (ENXIO); } +#if 0 /* * Reset PCI IODA table */ @@ -245,11 +283,32 @@ opalpci_attach(device_t dev) device_printf(dev, "IODA table reset failed: %d\n", err); return (ENXIO); } - while ((err = opal_call(OPAL_PCI_POLL, sc->phb_id)) > 0) - DELAY(1000*(err + 1)); /* Returns expected delay in ms */ + err = opal_call(OPAL_PCI_RESET, sc->phb_id, OPAL_RESET_PHB_COMPLETE, + 1); if (err < 0) { + device_printf(dev, "PHB reset failed: %d\n", err); + return (ENXIO); + } + if (err > 0) { + while ((err = opal_call(OPAL_PCI_POLL, sc->phb_id)) > 0) { + DELAY(1000*(err + 1)); /* Returns expected delay in ms */ + } + } + if (err < 0) { device_printf(dev, "WARNING: PHB IODA reset poll failed: %d\n", err); } + err = opal_call(OPAL_PCI_RESET, sc->phb_id, OPAL_RESET_PHB_COMPLETE, + 0); + if (err < 0) { + device_printf(dev, "PHB reset failed: %d\n", err); + return (ENXIO); + } + if (err > 0) { + while ((err = opal_call(OPAL_PCI_POLL, sc->phb_id)) > 0) { + DELAY(1000*(err + 1)); /* Returns expected delay in ms */ + } + } +#endif /* * Map all devices on the bus to partitionable endpoint one until @@ -275,49 +334,58 @@ opalpci_attach(device_t dev) device_printf(dev, "MMIO %d map failed: %d\n", i, err); } + if (OF_getencprop(node, "ibm,opal-available-m64-ranges", + m64ranges, sizeof(m64ranges)) == sizeof(m64ranges)) + m64bar = m64ranges[0]; + else + m64bar = 0; + /* XXX: multiple M64 windows? */ if (OF_getencprop(node, "ibm,opal-m64-window", m64window, sizeof(m64window)) == sizeof(m64window)) { opal_call(OPAL_PCI_PHB_MMIO_ENABLE, sc->phb_id, - OPAL_M64_WINDOW_TYPE, 0, 0); + OPAL_M64_WINDOW_TYPE, m64bar, 0); opal_call(OPAL_PCI_SET_PHB_MEM_WINDOW, sc->phb_id, - OPAL_M64_WINDOW_TYPE, 0 /* index */, + OPAL_M64_WINDOW_TYPE, m64bar /* index */, ((uint64_t)m64window[2] << 32) | m64window[3], 0, ((uint64_t)m64window[4] << 32) | m64window[5]); opal_call(OPAL_PCI_MAP_PE_MMIO_WINDOW, sc->phb_id, OPAL_PCI_DEFAULT_PE, OPAL_M64_WINDOW_TYPE, - 0 /* index */, 0); + m64bar /* index */, 0); opal_call(OPAL_PCI_PHB_MMIO_ENABLE, sc->phb_id, - OPAL_M64_WINDOW_TYPE, 0, OPAL_ENABLE_M64_NON_SPLIT); + OPAL_M64_WINDOW_TYPE, m64bar, OPAL_ENABLE_M64_NON_SPLIT); } /* * Enable IOMMU for PE1 - map everything 1:1 using - * segments of OPAL_PCI_TCE_SEG_SIZE size + * segments of max_tce_size size */ - maxmem = roundup2(powerpc_ptob(Maxmem), OPAL_PCI_TCE_SEG_SIZE); - entries = maxmem / OPAL_PCI_TCE_SEG_SIZE; + tce_size = max_tce_size(dev); + maxmem = roundup2(powerpc_ptob(Maxmem), tce_size); + entries = round_pow2(maxmem / tce_size); + tce_tbl_size = max(entries * sizeof(uint64_t), 4096); + device_printf(dev, "Entries: %ld, tce_size: %#lx\n", entries, tce_size); if (entries > OPAL_PCI_TCE_MAX_ENTRIES) panic("POWERNV supports only %jdGB of memory space\n", - (uintmax_t)((OPAL_PCI_TCE_MAX_ENTRIES * OPAL_PCI_TCE_SEG_SIZE) >> 30)); + (uintmax_t)((OPAL_PCI_TCE_MAX_ENTRIES * tce_size) >> 30)); if (bootverbose) device_printf(dev, "Mapping 0-%#jx for DMA\n", (uintmax_t)maxmem); - sc->tce = contigmalloc(OPAL_PCI_TCE_MAX_ENTRIES * sizeof(uint64_t), + sc->tce = contigmalloc(tce_tbl_size, M_DEVBUF, M_NOWAIT | M_ZERO, 0, - BUS_SPACE_MAXADDR_32BIT, OPAL_PCI_TCE_SEG_SIZE, 0); + BUS_SPACE_MAXADDR, tce_size, 0); if (sc->tce == NULL) panic("Failed to allocate TCE memory for PHB %jd\n", (uintmax_t)sc->phb_id); for (i = 0; i < entries; i++) - sc->tce[i] = (i * OPAL_PCI_TCE_SEG_SIZE) | OPAL_PCI_TCE_R | OPAL_PCI_TCE_W; + sc->tce[i] = (i * tce_size) | OPAL_PCI_TCE_R | OPAL_PCI_TCE_W; /* Map TCE for every PE. It seems necessary for Power8 */ for (i = 0; i < npe; i++) { err = opal_call(OPAL_PCI_MAP_PE_DMA_WINDOW, sc->phb_id, i, (i << 1), 1, pmap_kextract((uint64_t)&sc->tce[0]), - OPAL_PCI_TCE_MAX_ENTRIES * sizeof(uint64_t), OPAL_PCI_TCE_SEG_SIZE); + tce_tbl_size, tce_size); if (err != 0) { device_printf(dev, "DMA IOMMU mapping failed: %d\n", err); return (ENXIO);