From nobody Fri Nov 19 00:02:29 2021 X-Original-To: dev-commits-src-branches@mlmmj.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mlmmj.nyi.freebsd.org (Postfix) with ESMTP id EF152188F3FE; Fri, 19 Nov 2021 00:02:32 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "R3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4HwH0k51fJz3Blt; Fri, 19 Nov 2021 00:02:29 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id A10A41E53C; Fri, 19 Nov 2021 00:02:29 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 1AJ02TIf072308; Fri, 19 Nov 2021 00:02:29 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 1AJ02TiS072307; Fri, 19 Nov 2021 00:02:29 GMT (envelope-from git) Date: Fri, 19 Nov 2021 00:02:29 GMT Message-Id: <202111190002.1AJ02TiS072307@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org From: "Bjoern A. Zeeb" Subject: git: 37b2cf4e6a97 - stable/13 - LinuxKPI: implement dma_set_coherent_mask() List-Id: Commits to the stable branches of the FreeBSD src repository List-Archive: https://lists.freebsd.org/archives/dev-commits-src-branches List-Help: List-Post: List-Subscribe: List-Unsubscribe: Sender: owner-dev-commits-src-branches@freebsd.org X-BeenThere: dev-commits-src-branches@freebsd.org MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: bz X-Git-Repository: src X-Git-Refname: refs/heads/stable/13 X-Git-Reftype: branch X-Git-Commit: 37b2cf4e6a978dbea35009e361dfecab8efc0775 Auto-Submitted: auto-generated X-ThisMailContainsUnwantedMimeParts: N The branch stable/13 has been updated by bz: URL: https://cgit.FreeBSD.org/src/commit/?id=37b2cf4e6a978dbea35009e361dfecab8efc0775 commit 37b2cf4e6a978dbea35009e361dfecab8efc0775 Author: Bjoern A. Zeeb AuthorDate: 2021-09-27 22:50:07 +0000 Commit: Bjoern A. Zeeb CommitDate: 2021-11-19 00:01:24 +0000 LinuxKPI: implement dma_set_coherent_mask() Coherent is lower 32bit only by default in Linux and our only default dma mask is 64bit currently which violates expectations unless dma_set_coherent_mask() was called explicitly with a different mask. Implement coherent by creating a second tag, and storing the tags in the objects and use the tag from the object wherever possible. This currently does not update the scatterlist or pool (both could be converted but S/G cannot be MFCed as easily). There is a 2nd change embedded in the updated logic of linux_dma_alloc_coherent() to always zero the allocation as otherwise some drivers get cranky on uninialised garbage. Sponsored by: The FreeBSD Foundation (cherry picked from commit c39eefe715b3c835ce3d91a1c1932197c23c1f3c) (cherry picked from commit 1269873159c7fa0db3b9dbf8dadc54eec5dc0d58) --- .../linuxkpi/common/include/linux/dma-mapping.h | 7 +- sys/compat/linuxkpi/common/src/linux_pci.c | 194 +++++++++++++-------- 2 files changed, 129 insertions(+), 72 deletions(-) diff --git a/sys/compat/linuxkpi/common/include/linux/dma-mapping.h b/sys/compat/linuxkpi/common/include/linux/dma-mapping.h index 554d4bd4e695..c86a24d1270a 100644 --- a/sys/compat/linuxkpi/common/include/linux/dma-mapping.h +++ b/sys/compat/linuxkpi/common/include/linux/dma-mapping.h @@ -92,6 +92,7 @@ struct dma_map_ops { #define DMA_BIT_MASK(n) ((2ULL << ((n) - 1)) - 1ULL) int linux_dma_tag_init(struct device *, u64); +int linux_dma_tag_init_coherent(struct device *, u64); void *linux_dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag); dma_addr_t linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len); @@ -125,10 +126,10 @@ static inline int dma_set_coherent_mask(struct device *dev, u64 dma_mask) { - if (!dma_supported(dev, dma_mask)) + if (!dev->dma_priv || !dma_supported(dev, dma_mask)) return -EIO; - /* XXX Currently we don't support a separate coherent mask. */ - return 0; + + return (linux_dma_tag_init_coherent(dev, dma_mask)); } static inline int diff --git a/sys/compat/linuxkpi/common/src/linux_pci.c b/sys/compat/linuxkpi/common/src/linux_pci.c index c8f473205ede..780ba38d18dd 100644 --- a/sys/compat/linuxkpi/common/src/linux_pci.c +++ b/sys/compat/linuxkpi/common/src/linux_pci.c @@ -110,13 +110,31 @@ static device_method_t pci_methods[] = { struct linux_dma_priv { uint64_t dma_mask; - struct mtx lock; bus_dma_tag_t dmat; + uint64_t dma_coherent_mask; + bus_dma_tag_t dmat_coherent; + struct mtx lock; struct pctrie ptree; }; #define DMA_PRIV_LOCK(priv) mtx_lock(&(priv)->lock) #define DMA_PRIV_UNLOCK(priv) mtx_unlock(&(priv)->lock) +static int +linux_pdev_dma_uninit(struct pci_dev *pdev) +{ + struct linux_dma_priv *priv; + + priv = pdev->dev.dma_priv; + if (priv->dmat) + bus_dma_tag_destroy(priv->dmat); + if (priv->dmat_coherent) + bus_dma_tag_destroy(priv->dmat_coherent); + mtx_destroy(&priv->lock); + pdev->dev.dma_priv = NULL; + free(priv, M_DEVBUF); + return (0); +} + static int linux_pdev_dma_init(struct pci_dev *pdev) { @@ -124,34 +142,26 @@ linux_pdev_dma_init(struct pci_dev *pdev) int error; priv = malloc(sizeof(*priv), M_DEVBUF, M_WAITOK | M_ZERO); - pdev->dev.dma_priv = priv; mtx_init(&priv->lock, "lkpi-priv-dma", NULL, MTX_DEF); - pctrie_init(&priv->ptree); - /* create a default DMA tag */ + pdev->dev.dma_priv = priv; + + /* Create a default DMA tags. */ error = linux_dma_tag_init(&pdev->dev, DMA_BIT_MASK(64)); - if (error) { - mtx_destroy(&priv->lock); - free(priv, M_DEVBUF); - pdev->dev.dma_priv = NULL; - } - return (error); -} + if (error != 0) + goto err; + /* Coherent is lower 32bit only by default in Linux. */ + error = linux_dma_tag_init_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (error != 0) + goto err; -static int -linux_pdev_dma_uninit(struct pci_dev *pdev) -{ - struct linux_dma_priv *priv; + return (error); - priv = pdev->dev.dma_priv; - if (priv->dmat) - bus_dma_tag_destroy(priv->dmat); - mtx_destroy(&priv->lock); - free(priv, M_DEVBUF); - pdev->dev.dma_priv = NULL; - return (0); +err: + linux_pdev_dma_uninit(pdev); + return (error); } int @@ -185,6 +195,37 @@ linux_dma_tag_init(struct device *dev, u64 dma_mask) return (-error); } +int +linux_dma_tag_init_coherent(struct device *dev, u64 dma_mask) +{ + struct linux_dma_priv *priv; + int error; + + priv = dev->dma_priv; + + if (priv->dmat_coherent) { + if (priv->dma_coherent_mask == dma_mask) + return (0); + + bus_dma_tag_destroy(priv->dmat_coherent); + } + + priv->dma_coherent_mask = dma_mask; + + error = bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), + 1, 0, /* alignment, boundary */ + dma_mask, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filtfunc, filtfuncarg */ + BUS_SPACE_MAXSIZE, /* maxsize */ + 1, /* nsegments */ + BUS_SPACE_MAXSIZE, /* maxsegsz */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockfuncarg */ + &priv->dmat_coherent); + return (-error); +} + static struct pci_driver * linux_pci_find(device_t dev, const struct pci_device_id **idp) { @@ -723,6 +764,7 @@ struct linux_dma_obj { void *vaddr; uint64_t dma_addr; bus_dmamap_t dmamap; + bus_dma_tag_t dmat; }; static uma_zone_t linux_dma_trie_zone; @@ -768,44 +810,10 @@ linux_dma_trie_free(struct pctrie *ptree, void *node) PCTRIE_DEFINE(LINUX_DMA, linux_dma_obj, dma_addr, linux_dma_trie_alloc, linux_dma_trie_free); -void * -linux_dma_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag) -{ - struct linux_dma_priv *priv; - vm_paddr_t high; - size_t align; - void *mem; - - if (dev == NULL || dev->dma_priv == NULL) { - *dma_handle = 0; - return (NULL); - } - priv = dev->dma_priv; - if (priv->dma_mask) - high = priv->dma_mask; - else if (flag & GFP_DMA32) - high = BUS_SPACE_MAXADDR_32BIT; - else - high = BUS_SPACE_MAXADDR; - align = PAGE_SIZE << get_order(size); - mem = (void *)kmem_alloc_contig(size, flag & GFP_NATIVE_MASK, 0, high, - align, 0, VM_MEMATTR_DEFAULT); - if (mem != NULL) { - *dma_handle = linux_dma_map_phys(dev, vtophys(mem), size); - if (*dma_handle == 0) { - kmem_free((vm_offset_t)mem, size); - mem = NULL; - } - } else { - *dma_handle = 0; - } - return (mem); -} - #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) -dma_addr_t -linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len) +static dma_addr_t +linux_dma_map_phys_common(struct device *dev, vm_paddr_t phys, size_t len, + bus_dma_tag_t dmat) { struct linux_dma_priv *priv; struct linux_dma_obj *obj; @@ -820,25 +828,26 @@ linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len) * bus_dma API. This avoids tracking collisions in the pctrie * with the additional benefit of reducing overhead. */ - if (bus_dma_id_mapped(priv->dmat, phys, len)) + if (bus_dma_id_mapped(dmat, phys, len)) return (phys); obj = uma_zalloc(linux_dma_obj_zone, M_NOWAIT); if (obj == NULL) { return (0); } + obj->dmat = dmat; DMA_PRIV_LOCK(priv); - if (bus_dmamap_create(priv->dmat, 0, &obj->dmamap) != 0) { + if (bus_dmamap_create(obj->dmat, 0, &obj->dmamap) != 0) { DMA_PRIV_UNLOCK(priv); uma_zfree(linux_dma_obj_zone, obj); return (0); } nseg = -1; - if (_bus_dmamap_load_phys(priv->dmat, obj->dmamap, phys, len, + if (_bus_dmamap_load_phys(obj->dmat, obj->dmamap, phys, len, BUS_DMA_NOWAIT, &seg, &nseg) != 0) { - bus_dmamap_destroy(priv->dmat, obj->dmamap); + bus_dmamap_destroy(obj->dmat, obj->dmamap); DMA_PRIV_UNLOCK(priv); uma_zfree(linux_dma_obj_zone, obj); return (0); @@ -849,8 +858,8 @@ linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len) error = LINUX_DMA_PCTRIE_INSERT(&priv->ptree, obj); if (error != 0) { - bus_dmamap_unload(priv->dmat, obj->dmamap); - bus_dmamap_destroy(priv->dmat, obj->dmamap); + bus_dmamap_unload(obj->dmat, obj->dmamap); + bus_dmamap_destroy(obj->dmat, obj->dmamap); DMA_PRIV_UNLOCK(priv); uma_zfree(linux_dma_obj_zone, obj); return (0); @@ -859,13 +868,23 @@ linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len) return (obj->dma_addr); } #else -dma_addr_t -linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len) +static dma_addr_t +linux_dma_map_phys_common(struct device *dev __unused, vm_paddr_t phys, + size_t len __unused, bus_dma_tag_t dmat __unused) { return (phys); } #endif +dma_addr_t +linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len) +{ + struct linux_dma_priv *priv; + + priv = dev->dma_priv; + return (linux_dma_map_phys_common(dev, phys, len, priv->dmat)); +} + #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) void linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len) @@ -885,8 +904,8 @@ linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len) return; } LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, dma_addr); - bus_dmamap_unload(priv->dmat, obj->dmamap); - bus_dmamap_destroy(priv->dmat, obj->dmamap); + bus_dmamap_unload(obj->dmat, obj->dmamap); + bus_dmamap_destroy(obj->dmat, obj->dmamap); DMA_PRIV_UNLOCK(priv); uma_zfree(linux_dma_obj_zone, obj); @@ -898,6 +917,43 @@ linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len) } #endif +void * +linux_dma_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t flag) +{ + struct linux_dma_priv *priv; + vm_paddr_t high; + size_t align; + void *mem; + + if (dev == NULL || dev->dma_priv == NULL) { + *dma_handle = 0; + return (NULL); + } + priv = dev->dma_priv; + if (priv->dma_coherent_mask) + high = priv->dma_coherent_mask; + else + /* Coherent is lower 32bit only by default in Linux. */ + high = BUS_SPACE_MAXADDR_32BIT; + align = PAGE_SIZE << get_order(size); + /* Always zero the allocation. */ + flag |= M_ZERO; + mem = (void *)kmem_alloc_contig(size, flag & GFP_NATIVE_MASK, 0, high, + align, 0, VM_MEMATTR_DEFAULT); + if (mem != NULL) { + *dma_handle = linux_dma_map_phys_common(dev, vtophys(mem), size, + priv->dmat_coherent); + if (*dma_handle == 0) { + kmem_free((vm_offset_t)mem, size); + mem = NULL; + } + } else { + *dma_handle = 0; + } + return (mem); +} + int linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction dir __unused, unsigned long attrs __unused)