Date: Wed, 2 Mar 2016 08:39:59 +0000 (UTC) From: Wojciech Macek <wma@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r296307 - head/sys/arm64/cavium Message-ID: <201603020839.u228dxvG028785@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: wma Date: Wed Mar 2 08:39:59 2016 New Revision: 296307 URL: https://svnweb.freebsd.org/changeset/base/296307 Log: Improve ThunderX PEM driver to work on pass2 revision Things changed: * do not allocate 4GB of SLI space, because it's the waste of system resources. Allocate only small portions when needed. * provide own implementation of activate_resource which performs address translation between PCI bus and host PA address space. This is temporary solution, should be replaced by bus_map_resource once implemented. Obtained from: Semihalf Sponsored by: Cavium Approved by: cognet (mentor) Reviewed by: jhb Differential revision: https://reviews.freebsd.org/D5294 Modified: head/sys/arm64/cavium/thunder_pcie_common.c head/sys/arm64/cavium/thunder_pcie_common.h head/sys/arm64/cavium/thunder_pcie_pem.c Modified: head/sys/arm64/cavium/thunder_pcie_common.c ============================================================================== --- head/sys/arm64/cavium/thunder_pcie_common.c Wed Mar 2 05:43:16 2016 (r296306) +++ head/sys/arm64/cavium/thunder_pcie_common.c Wed Mar 2 08:39:59 2016 (r296307) @@ -106,6 +106,28 @@ range_addr_is_phys(struct pcie_range *ra } uint64_t +range_addr_phys_to_pci(struct pcie_range *ranges, uint64_t phys_addr) +{ + struct pcie_range *r; + uint64_t offset; + int tuple; + + /* Find physical address corresponding to given bus address */ + for (tuple = 0; tuple < MAX_RANGES_TUPLES; tuple++) { + r = &ranges[tuple]; + if (phys_addr >= r->phys_base && + phys_addr < (r->phys_base + r->size)) { + /* Given phys addr is in this range. + * Translate phys addr to bus addr. + */ + offset = phys_addr - r->phys_base; + return (r->pci_base + offset); + } + } + return (0); +} + +uint64_t range_addr_pci_to_phys(struct pcie_range *ranges, uint64_t pci_addr) { struct pcie_range *r; Modified: head/sys/arm64/cavium/thunder_pcie_common.h ============================================================================== --- head/sys/arm64/cavium/thunder_pcie_common.h Wed Mar 2 05:43:16 2016 (r296306) +++ head/sys/arm64/cavium/thunder_pcie_common.h Wed Mar 2 08:39:59 2016 (r296307) @@ -36,6 +36,7 @@ MALLOC_DECLARE(M_THUNDER_PCIE); uint32_t range_addr_is_pci(struct pcie_range *, uint64_t, uint64_t); uint32_t range_addr_is_phys(struct pcie_range *, uint64_t, uint64_t); +uint64_t range_addr_phys_to_pci(struct pcie_range *, uint64_t); uint64_t range_addr_pci_to_phys(struct pcie_range *, uint64_t); int thunder_pcie_identify_ecam(device_t, int *); Modified: head/sys/arm64/cavium/thunder_pcie_pem.c ============================================================================== --- head/sys/arm64/cavium/thunder_pcie_pem.c Wed Mar 2 05:43:16 2016 (r296306) +++ head/sys/arm64/cavium/thunder_pcie_pem.c Wed Mar 2 08:39:59 2016 (r296307) @@ -93,7 +93,6 @@ __FBSDID("$FreeBSD$"); #define SLIX_S2M_REGX_ACC_SPACING 0x001000000000UL #define SLI_BASE 0x880000000000UL #define SLI_WINDOW_SPACING 0x004000000000UL -#define SLI_WINDOW_SIZE 0x0000FF000000UL #define SLI_PCI_OFFSET 0x001000000000UL #define SLI_NODE_SHIFT (44) #define SLI_NODE_MASK (3) @@ -121,9 +120,15 @@ __FBSDID("$FreeBSD$"); #define RID_PEM_SPACE 1 +static int thunder_pem_activate_resource(device_t, device_t, int, int, + struct resource *); +static int thunder_pem_adjust_resource(device_t, device_t, int, + struct resource *, rman_res_t, rman_res_t); static struct resource * thunder_pem_alloc_resource(device_t, device_t, int, int *, rman_res_t, rman_res_t, rman_res_t, u_int); static int thunder_pem_attach(device_t); +static int thunder_pem_deactivate_resource(device_t, device_t, int, int, + struct resource *); static int thunder_pem_detach(device_t); static uint64_t thunder_pem_config_reg_read(struct thunder_pem_softc *, int); static int thunder_pem_link_init(struct thunder_pem_softc *); @@ -135,6 +140,7 @@ static int thunder_pem_read_ivar(device_ static void thunder_pem_release_all(device_t); static int thunder_pem_release_resource(device_t, device_t, int, int, struct resource *); +static struct rman * thunder_pem_rman(struct thunder_pem_softc *, int); static void thunder_pem_slix_s2m_regx_acc_modify(struct thunder_pem_softc *, int, int); static void thunder_pem_write_config(device_t, u_int, u_int, u_int, u_int, @@ -157,8 +163,9 @@ static device_method_t thunder_pem_metho DEVMETHOD(bus_write_ivar, thunder_pem_write_ivar), DEVMETHOD(bus_alloc_resource, thunder_pem_alloc_resource), DEVMETHOD(bus_release_resource, thunder_pem_release_resource), - DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), - DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), + DEVMETHOD(bus_adjust_resource, thunder_pem_adjust_resource), + DEVMETHOD(bus_activate_resource, thunder_pem_activate_resource), + DEVMETHOD(bus_deactivate_resource, thunder_pem_deactivate_resource), DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), @@ -174,6 +181,7 @@ DEFINE_CLASS_0(pcib, thunder_pem_driver, sizeof(struct thunder_pem_softc)); static devclass_t thunder_pem_devclass; +extern struct bus_space memmap_bus; DRIVER_MODULE(thunder_pem, pci, thunder_pem_driver, thunder_pem_devclass, 0, 0); MODULE_DEPEND(thunder_pem, pci, 1, 1, 1); @@ -225,6 +233,88 @@ thunder_pem_write_ivar(device_t dev, dev } static int +thunder_pem_activate_resource(device_t dev, device_t child, int type, int rid, + struct resource *r) +{ + int err; + bus_addr_t paddr; + bus_size_t psize; + bus_space_handle_t vaddr; + struct thunder_pem_softc *sc; + + if ((err = rman_activate_resource(r)) != 0) + return (err); + + sc = device_get_softc(dev); + + /* + * If this is a memory resource, map it into the kernel. + */ + if (type == SYS_RES_MEMORY || type == SYS_RES_IOPORT) { + paddr = (bus_addr_t)rman_get_start(r); + psize = (bus_size_t)rman_get_size(r); + + paddr = range_addr_pci_to_phys(sc->ranges, paddr); + + err = bus_space_map(&memmap_bus, paddr, psize, 0, &vaddr); + if (err != 0) { + rman_deactivate_resource(r); + return (err); + } + rman_set_bustag(r, &memmap_bus); + rman_set_virtual(r, (void *)vaddr); + rman_set_bushandle(r, vaddr); + } + return (0); +} + +/* + * This function is an exact copy of nexus_deactivate_resource() + * Keep it up-to-date with all changes in nexus. To be removed + * once bus-mapping interface is developed. + */ +static int +thunder_pem_deactivate_resource(device_t bus, device_t child, int type, int rid, + struct resource *r) +{ + bus_size_t psize; + bus_space_handle_t vaddr; + + psize = (bus_size_t)rman_get_size(r); + vaddr = rman_get_bushandle(r); + + if (vaddr != 0) { + bus_space_unmap(&memmap_bus, vaddr, psize); + rman_set_virtual(r, NULL); + rman_set_bushandle(r, 0); + } + + return (rman_deactivate_resource(r)); +} + +static int +thunder_pem_adjust_resource(device_t dev, device_t child, int type, + struct resource *res, rman_res_t start, rman_res_t end) +{ + struct thunder_pem_softc *sc; + struct rman *rm; + + sc = device_get_softc(dev); + + rm = thunder_pem_rman(sc, type); + if (rm == NULL) + return (bus_generic_adjust_resource(dev, child, type, res, + start, end)); + if (!rman_is_region_manager(res, rm)) + /* + * This means a child device has a memory or I/O + * resource not from you which shouldn't happen. + */ + return (EINVAL); + return (rman_adjust_resource(res, start, end)); +} + +static int thunder_pem_identify(device_t dev) { struct thunder_pem_softc *sc; @@ -314,14 +404,6 @@ thunder_pem_init(struct thunder_pem_soft return retval; } - retval = bus_space_map(sc->reg_bst, sc->sli_window_base, - SLI_WINDOW_SIZE, 0, &sc->pem_sli_base); - if (retval) { - device_printf(sc->dev, - "Unable to map RC%d pem_addr base address", sc->id); - return (ENOMEM); - } - /* To support 32-bit PCIe devices, set S2M_REGx_ACC[BA]=0x0 */ for (i = 0; i < SLI_ACC_REG_CNT; i++) { thunder_pem_slix_s2m_regx_acc_modify(sc, sc->sli_group, i); @@ -365,24 +447,30 @@ thunder_pem_read_config(device_t dev, u_ /* Calculate offset */ offset = (bus << PEM_BUS_SHIFT) | (slot << PEM_SLOT_SHIFT) | - (func << PEM_FUNC_SHIFT) | reg; + (func << PEM_FUNC_SHIFT); t = sc->reg_bst; h = sc->pem_sli_base; + bus_space_map(sc->reg_bst, sc->sli_window_base + offset, + PCIE_REGMAX, 0, &h); + switch (bytes) { case 1: - data = bus_space_read_1(t, h, offset); + data = bus_space_read_1(t, h, reg); break; case 2: - data = le16toh(bus_space_read_2(t, h, offset)); + data = le16toh(bus_space_read_2(t, h, reg)); break; case 4: - data = le32toh(bus_space_read_4(t, h, offset)); + data = le32toh(bus_space_read_4(t, h, reg)); break; default: - return (~0U); + data = ~0U; + break; } + bus_space_unmap(sc->reg_bst, h, PCIE_REGMAX); + return (data); } @@ -403,23 +491,28 @@ thunder_pem_write_config(device_t dev, u /* Calculate offset */ offset = (bus << PEM_BUS_SHIFT) | (slot << PEM_SLOT_SHIFT) | - (func << PEM_FUNC_SHIFT) | reg; + (func << PEM_FUNC_SHIFT); t = sc->reg_bst; h = sc->pem_sli_base; + bus_space_map(sc->reg_bst, sc->sli_window_base + offset, + PCIE_REGMAX, 0, &h); + switch (bytes) { case 1: - bus_space_write_1(t, h, offset, val); + bus_space_write_1(t, h, reg, val); break; case 2: - bus_space_write_2(t, h, offset, htole16(val)); + bus_space_write_2(t, h, reg, htole16(val)); break; case 4: - bus_space_write_4(t, h, offset, htole32(val)); + bus_space_write_4(t, h, reg, htole32(val)); break; default: - return; + break; } + + bus_space_unmap(sc->reg_bst, h, PCIE_REGMAX); } static struct resource * @@ -431,35 +524,30 @@ thunder_pem_alloc_resource(device_t dev, struct resource *res; device_t parent_dev; - switch (type) { - case SYS_RES_IOPORT: - rm = &sc->io_rman; - break; - case SYS_RES_MEMORY: - rm = &sc->mem_rman; - break; - default: + rm = thunder_pem_rman(sc, type); + if (rm == NULL) { /* Find parent device. On ThunderX we know an exact path. */ parent_dev = device_get_parent(device_get_parent(dev)); return (BUS_ALLOC_RESOURCE(parent_dev, dev, type, rid, start, end, count, flags)); }; - if (RMAN_IS_DEFAULT_RANGE(start, end)) { - device_printf(dev, - "Cannot allocate resource with unspecified range\n"); - goto fail; - } - /* Translate PCI address to host PHYS */ - if (range_addr_is_pci(sc->ranges, start, count) == 0) - goto fail; - start = range_addr_pci_to_phys(sc->ranges, start); - end = start + count - 1; + if (!RMAN_IS_DEFAULT_RANGE(start, end)) { + /* + * We might get PHYS addresses here inherited from EFI. + * Convert to PCI if necessary. + */ + if (range_addr_is_phys(sc->ranges, start, count)) { + start = range_addr_phys_to_pci(sc->ranges, start); + end = start + count - 1; + } + + } if (bootverbose) { device_printf(dev, - "rman_reserve_resource: start=%#lx, end=%#lx, count=%#lx\n", + "thunder_pem_alloc_resource: start=%#lx, end=%#lx, count=%#lx\n", start, end, count); } @@ -503,6 +591,22 @@ thunder_pem_release_resource(device_t de return (rman_release_resource(res)); } +static struct rman * +thunder_pem_rman(struct thunder_pem_softc *sc, int type) +{ + + switch (type) { + case SYS_RES_IOPORT: + return (&sc->io_rman); + case SYS_RES_MEMORY: + return (&sc->mem_rman); + default: + break; + } + + return (NULL); +} + static int thunder_pem_probe(device_t dev) { @@ -529,6 +633,9 @@ thunder_pem_attach(device_t dev) struct thunder_pem_softc *sc; int error; int rid; + int tuple; + uint64_t base, size; + struct rman *rman; sc = device_get_softc(dev); sc->dev = dev; @@ -601,16 +708,42 @@ thunder_pem_attach(device_t dev) sc->ranges[0].size = PCI_MEMORY_SIZE; sc->ranges[0].phys_base = sc->sli_window_base + SLI_PCI_OFFSET + sc->ranges[0].pci_base; - rman_manage_region(&sc->mem_rman, sc->ranges[0].phys_base, - sc->ranges[0].phys_base + sc->ranges[0].size - 1); + sc->ranges[0].flags = SYS_RES_MEMORY; /* Fill IO window */ sc->ranges[1].pci_base = PCI_IO_BASE; sc->ranges[1].size = PCI_IO_SIZE; sc->ranges[1].phys_base = sc->sli_window_base + SLI_PCI_OFFSET + sc->ranges[1].pci_base; - rman_manage_region(&sc->io_rman, sc->ranges[1].phys_base, - sc->ranges[1].phys_base + sc->ranges[1].size - 1); + sc->ranges[1].flags = SYS_RES_IOPORT; + + for (tuple = 0; tuple < MAX_RANGES_TUPLES; tuple++) { + base = sc->ranges[tuple].pci_base; + size = sc->ranges[tuple].size; + if (size == 0) + continue; /* empty range element */ + + rman = thunder_pem_rman(sc, sc->ranges[tuple].flags); + if (rman != NULL) + error = rman_manage_region(rman, base, + base + size - 1); + else + error = EINVAL; + if (error) { + device_printf(dev, + "rman_manage_region() failed. error = %d\n", error); + rman_fini(&sc->mem_rman); + return (error); + } + if (bootverbose) { + device_printf(dev, + "\tPCI addr: 0x%jx, CPU addr: 0x%jx, Size: 0x%jx, Flags:0x%jx\n", + sc->ranges[tuple].pci_base, + sc->ranges[tuple].phys_base, + sc->ranges[tuple].size, + sc->ranges[tuple].flags); + } + } if (thunder_pem_init(sc)) { device_printf(dev, "Failure during PEM init\n");
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201603020839.u228dxvG028785>